Related
I am working with Unity 4.5, grabbing images as bytes arrays (each byte represent a channel, taking 4 bytes per pixel (rgba) and displaying them on a texture converting the array to a Color32 array, using this loop:
img = new Color32[byteArray.Length / nChannels]; //nChannels being 4
for (int i=0; i< img.Length; i++) {
img[i].r = byteArray[i*nChannels];
img[i].g = byteArray[i*nChannels+1];
img[i].b = byteArray[i*nChannels+2];
img[i].a = byteArray[i*nChannels+3];
}
Then, it is applied to the texture using:
tex.SetPixels32(img);
However, this slows down the application significantly (this loop is executed on every single frame), and I would like to know if there is any other way to speed up the copying process. I've found some people (Fast copy of Color32[] array to byte[] array) using the Marshal.Copy functions in order to do the reverse process (Color32 to byte array), but I have not been able to make it work to copy a byte array to a Color32 array. Does anybody know a faster way?
Thank you in advance!
Yes, Marshal.Copy is the way to go. I've answered a similar question here.
Here's a generic method to copy from struct[] to byte[] and vice versa
private static byte[] ToByteArray<T>(T[] source) where T : struct
{
GCHandle handle = GCHandle.Alloc(source, GCHandleType.Pinned);
try
{
IntPtr pointer = handle.AddrOfPinnedObject();
byte[] destination = new byte[source.Length * Marshal.SizeOf(typeof(T))];
Marshal.Copy(pointer, destination, 0, destination.Length);
return destination;
}
finally
{
if (handle.IsAllocated)
handle.Free();
}
}
private static T[] FromByteArray<T>(byte[] source) where T : struct
{
T[] destination = new T[source.Length / Marshal.SizeOf(typeof(T))];
GCHandle handle = GCHandle.Alloc(destination, GCHandleType.Pinned);
try
{
IntPtr pointer = handle.AddrOfPinnedObject();
Marshal.Copy(source, 0, pointer, source.Length);
return destination;
}
finally
{
if (handle.IsAllocated)
handle.Free();
}
}
Use it as:
[StructLayout(LayoutKind.Sequential)]
public struct Demo
{
public double X;
public double Y;
}
private static void Main()
{
Demo[] array = new Demo[2];
array[0] = new Demo { X = 5.6, Y = 6.6 };
array[1] = new Demo { X = 7.6, Y = 8.6 };
byte[] bytes = ToByteArray(array);
Demo[] array2 = FromByteArray<Demo>(bytes);
}
This code requires unsafe switch but should be fast. I think you should benchmark these answers...
var bytes = new byte[] { 1, 2, 3, 4 };
var colors = MemCopyUtils.ByteArrayToColor32Array(bytes);
public class MemCopyUtils
{
unsafe delegate void MemCpyDelegate(byte* dst, byte* src, int len);
static MemCpyDelegate MemCpy;
static MemCopyUtils()
{
InitMemCpy();
}
static void InitMemCpy()
{
var mi = typeof(Buffer).GetMethod(
name: "Memcpy",
bindingAttr: BindingFlags.NonPublic | BindingFlags.Static,
binder: null,
types: new Type[] { typeof(byte*), typeof(byte*), typeof(int) },
modifiers: null);
MemCpy = (MemCpyDelegate)Delegate.CreateDelegate(typeof(MemCpyDelegate), mi);
}
public unsafe static Color32[] ByteArrayToColor32Array(byte[] bytes)
{
Color32[] colors = new Color32[bytes.Length / sizeof(Color32)];
fixed (void* tempC = &colors[0])
fixed (byte* pBytes = bytes)
{
byte* pColors = (byte*)tempC;
MemCpy(pColors, pBytes, bytes.Length);
}
return colors;
}
}
Using Parallel.For may give you a significant performance increase.
img = new Color32[byteArray.Length / nChannels]; //nChannels being 4
Parallel.For(0, img.Length, i =>
{
img[i].r = byteArray[i*nChannels];
img[i].g = byteArray[i*nChannels+1];
img[i].b = byteArray[i*nChannels+2];
img[i].a = byteArray[i*nChannels+3];
});
Example on MSDN
I haven't profiled it, but using fixed to ensure your memory doesn't get moved around and to remove bounds checks on array accesses might provide some benefit:
img = new Color32[byteArray.Length / nChannels]; //nChannels being 4
fixed (byte* ba = byteArray)
{
fixed (Color32* c = img)
{
byte* byteArrayPtr = ba;
Color32* colorPtr = c;
for (int i = 0; i < img.Length; i++)
{
(*colorPtr).r = *byteArrayPtr++;
(*colorPtr).g = *byteArrayPtr++;
(*colorPtr).b = *byteArrayPtr++;
(*colorPtr).a = *byteArrayPtr++;
colorPtr++;
}
}
}
It might not provide much more benefit on 64-bit systems - I believe that the bounds checking is is more highly optimized. Also, this is an unsafe operation, so take care.
public Color32[] GetColorArray(byte[] myByte)
{
if (myByte.Length % 1 != 0)
throw new Exception("Must have an even length");
var colors = new Color32[myByte.Length / nChannels];
for (var i = 0; i < myByte.Length; i += nChannels)
{
colors[i / nChannels] = new Color32(
(byte)(myByte[i] & 0xF8),
(byte)(((myByte[i] & 7) << 5) | ((myByte[i + 1] & 0xE0) >> 3)),
(byte)((myByte[i + 1] & 0x1F) << 3),
(byte)1);
}
return colors;
}
Worked about 30-50 times faster than just i++. The "extras" is just styling. This code is doing, in one "line", in the for loop, what you're declaring in 4 lines plus it is much quicker. Cheers :)
Referenced + Referenced code: Here
How do I convert a structure to a byte array in C#?
I have defined a structure like this:
public struct CIFSPacket
{
public uint protocolIdentifier; //The value must be "0xFF+'SMB'".
public byte command;
public byte errorClass;
public byte reserved;
public ushort error;
public byte flags;
//Here there are 14 bytes of data which is used differently among different dialects.
//I do want the flags2. However, so I'll try parsing them.
public ushort flags2;
public ushort treeId;
public ushort processId;
public ushort userId;
public ushort multiplexId;
//Trans request
public byte wordCount;//Count of parameter words defining the data portion of the packet.
//From here it might be undefined...
public int parametersStartIndex;
public ushort byteCount; //Buffer length
public int bufferStartIndex;
public string Buffer;
}
In my main method, I create an instance of it and assign values to it:
CIFSPacket packet = new CIFSPacket();
packet.protocolIdentifier = 0xff;
packet.command = (byte)CommandTypes.SMB_COM_NEGOTIATE;
packet.errorClass = 0xff;
packet.error = 0;
packet.flags = 0x00;
packet.flags2 = 0x0001;
packet.multiplexId = 22;
packet.wordCount = 0;
packet.byteCount = 119;
packet.Buffer = "NT LM 0.12";
Now I want to send this Packet by socket. For that, I need to convert the structure to a byte array. How can I do it?
My full code is as follows.
static void Main(string[] args)
{
Socket MyPing = new Socket(AddressFamily.InterNetwork,
SocketType.Stream , ProtocolType.Unspecified ) ;
MyPing.Connect("172.24.18.240", 139);
//Fake an IP Address so I can send with SendTo
IPAddress IP = new IPAddress(new byte[] { 172,24,18,240 });
IPEndPoint IPEP = new IPEndPoint(IP, 139);
//Local IP for Receiving
IPEndPoint Local = new IPEndPoint(IPAddress.Any, 0);
EndPoint EP = (EndPoint)Local;
CIFSPacket packet = new CIFSPacket();
packet.protocolIdentifier = 0xff;
packet.command = (byte)CommandTypes.SMB_COM_NEGOTIATE;
packet.errorClass = 0xff;
packet.error = 0;
packet.flags = 0x00;
packet.flags2 = 0x0001;
packet.multiplexId = 22;
packet.wordCount = 0;
packet.byteCount = 119;
packet.Buffer = "NT LM 0.12";
MyPing.SendTo(It takes byte array as parameter);
}
What would a code snippet be?
This is fairly easy, using marshalling.
Top of file
using System.Runtime.InteropServices
Function
byte[] getBytes(CIFSPacket str) {
int size = Marshal.SizeOf(str);
byte[] arr = new byte[size];
IntPtr ptr = IntPtr.Zero;
try
{
ptr = Marshal.AllocHGlobal(size);
Marshal.StructureToPtr(str, ptr, true);
Marshal.Copy(ptr, arr, 0, size);
}
finally
{
Marshal.FreeHGlobal(ptr);
}
return arr;
}
And to convert it back:
CIFSPacket fromBytes(byte[] arr)
{
CIFSPacket str = new CIFSPacket();
int size = Marshal.SizeOf(str);
IntPtr ptr = IntPtr.Zero;
try
{
ptr = Marshal.AllocHGlobal(size);
Marshal.Copy(arr, 0, ptr, size);
str = (CIFSPacket)Marshal.PtrToStructure(ptr, str.GetType());
}
finally
{
Marshal.FreeHGlobal(ptr);
}
return str;
}
In your structure, you will need to put this before a string
[MarshalAs(UnmanagedType.ByValTStr, SizeConst = 100)]
public string Buffer;
And make sure SizeConst is as big as your biggest possible string.
And you should probably read this:
http://msdn.microsoft.com/en-us/library/4ca6d5z7.aspx
If you really want it to be FAST on Windows, you can do it using unsafe code with CopyMemory. CopyMemory is about 5x faster (e.g. 800MB of data takes 3s to copy via marshalling, while only taking .6s to copy via CopyMemory). This method does limit you to using only data which is actually stored in the struct blob itself, e.g. numbers, or fixed length byte arrays.
[DllImport("kernel32.dll", EntryPoint = "CopyMemory", SetLastError = false)]
private static unsafe extern void CopyMemory(void *dest, void *src, int count);
private static unsafe byte[] Serialize(TestStruct[] index)
{
var buffer = new byte[Marshal.SizeOf(typeof(TestStruct)) * index.Length];
fixed (void* d = &buffer[0])
{
fixed (void* s = &index[0])
{
CopyMemory(d, s, buffer.Length);
}
}
return buffer;
}
Have a look at these methods:
byte [] StructureToByteArray(object obj)
{
int len = Marshal.SizeOf(obj);
byte [] arr = new byte[len];
IntPtr ptr = Marshal.AllocHGlobal(len);
Marshal.StructureToPtr(obj, ptr, true);
Marshal.Copy(ptr, arr, 0, len);
Marshal.FreeHGlobal(ptr);
return arr;
}
void ByteArrayToStructure(byte [] bytearray, ref object obj)
{
int len = Marshal.SizeOf(obj);
IntPtr i = Marshal.AllocHGlobal(len);
Marshal.Copy(bytearray,0, i,len);
obj = Marshal.PtrToStructure(i, obj.GetType());
Marshal.FreeHGlobal(i);
}
This is a shameless copy of another thread which I found upon Googling!
Update : For more details, check the source
Variant of the code of Vicent with one less memory allocation:
public static byte[] GetBytes<T>(T str)
{
int size = Marshal.SizeOf(str);
byte[] arr = new byte[size];
GCHandle h = default(GCHandle);
try
{
h = GCHandle.Alloc(arr, GCHandleType.Pinned);
Marshal.StructureToPtr<T>(str, h.AddrOfPinnedObject(), false);
}
finally
{
if (h.IsAllocated)
{
h.Free();
}
}
return arr;
}
public static T FromBytes<T>(byte[] arr) where T : struct
{
T str = default(T);
GCHandle h = default(GCHandle);
try
{
h = GCHandle.Alloc(arr, GCHandleType.Pinned);
str = Marshal.PtrToStructure<T>(h.AddrOfPinnedObject());
}
finally
{
if (h.IsAllocated)
{
h.Free();
}
}
return str;
}
I use GCHandle to "pin" the memory and then I use directly its address with h.AddrOfPinnedObject().
I know this is really late, but with C# 7.3 you can do this for unmanaged structs or anything else that's unmanged (int, bool etc...):
public static unsafe byte[] ConvertToBytes<T>(T value) where T : unmanaged {
byte* pointer = (byte*)&value;
byte[] bytes = new byte[sizeof(T)];
for (int i = 0; i < sizeof(T); i++) {
bytes[i] = pointer[i];
}
return bytes;
}
Then use like this:
struct MyStruct {
public int Value1;
public int Value2;
//.. blah blah blah
}
byte[] bytes = ConvertToBytes(new MyStruct());
As the main answer is using CIFSPacket type, which is not (or no longer) available in C#, I wrote correct methods:
static byte[] getBytes(object str)
{
int size = Marshal.SizeOf(str);
byte[] arr = new byte[size];
IntPtr ptr = Marshal.AllocHGlobal(size);
Marshal.StructureToPtr(str, ptr, true);
Marshal.Copy(ptr, arr, 0, size);
Marshal.FreeHGlobal(ptr);
return arr;
}
static T fromBytes<T>(byte[] arr)
{
T str = default(T);
int size = Marshal.SizeOf(str);
IntPtr ptr = Marshal.AllocHGlobal(size);
Marshal.Copy(arr, 0, ptr, size);
str = (T)Marshal.PtrToStructure(ptr, str.GetType());
Marshal.FreeHGlobal(ptr);
return str;
}
Tested, they work.
You can use Marshal (StructureToPtr, ptrToStructure), and Marshal.copy but this is plataform dependent.
Serialization includes Functions to Custom Serialization.
public virtual void GetObjectData(SerializationInfo info, StreamingContext context)
Protected Sub New(ByVal info As SerializationInfo, ByVal context As StreamingContext)
SerializationInfo include functions to serialize each member.
BinaryWriter and BinaryReader also contains methods to Save / Load to Byte Array (Stream).
Note that you can create a MemoryStream from a Byte Array or a Byte Array from a MemoryStream.
You can create a method Save and a method New on your structure:
Save(Bw as BinaryWriter)
New (Br as BinaryReader)
Then you select members to Save / Load to Stream -> Byte Array.
Almost all of the answers here use Marshal.StructureToPtr, which might be good for P/Invoke but it is very slow, and doesn't even always represent the actual raw content of the value. #Varscott128's answer is much better but it also contains an explicit byte copying, which is not necessary.
For unmanaged structs (structs without managed references) all you need is to reinterpret the allocated result array so a simple assignment does the trick (works even for huge structs):
.NET (Core) Solution:
If you can use the Unsafe class, then the solution is really easy. The unsafe modifier is required only due to sizeof(T).
public static unsafe byte[] SerializeValueType<T>(in T value) where T : unmanaged
{
byte[] result = new byte[sizeof(T)];
Unsafe.As<byte, T>(ref result[0]) = value;
return result;
}
// Note: Validation is omitted for simplicity
public static T DeserializeValueType<T>(byte[] data) where T : unmanaged
=> return Unsafe.As<byte, T>(ref data[0]);
.NET Framework/Standard Solution:
public static unsafe byte[] SerializeValueType<T>(in T value) where T : unmanaged
{
byte[] result = new byte[sizeof(T)];
fixed (byte* dst = result)
*(T*)dst = value;
return result;
}
// Note: Validation is omitted for simplicity
public static unsafe T DeserializeValueType<T>(byte[] data) where T : unmanaged
{
fixed (byte* src = data)
return *(T*)src;
}
See the complete code with validations here.
Remarks:
The OP's example contains a string, which is a reference type so the solution above cannot be used for that. And if you can't use generic methods for some reason things start to get more complicated, especially for .NET Framework (but non-generic size calculation is a pain also on the Core platform). If performance does not matter, then you can revert to Marshal.SizeOf and StructureToPtr as suggested by several other answers, or feel free to use the BinarySerializer.SerializeValueType method from my library that I linked also for the examples above (NuGet).
This can be done very straightforwardly.
Define your struct explicitly with [StructLayout(LayoutKind.Explicit)]
int size = list.GetLength(0);
IntPtr addr = Marshal.AllocHGlobal(size * sizeof(DataStruct));
DataStruct *ptrBuffer = (DataStruct*)addr;
foreach (DataStruct ds in list)
{
*ptrBuffer = ds;
ptrBuffer += 1;
}
This code can only be written in an unsafe context. You have to free addr when you're done with it.
Marshal.FreeHGlobal(addr);
I've come up with a different approach that could convert any struct without the hassle of fixing length, however the resulting byte array would have a little bit more overhead.
Here is a sample struct:
[StructLayout(LayoutKind.Sequential)]
public class HelloWorld
{
public MyEnum enumvalue;
public string reqtimestamp;
public string resptimestamp;
public string message;
public byte[] rawresp;
}
As you can see, all those structures would require adding the fixed length attributes. Which could often ended up taking up more space than required. Note that the LayoutKind.Sequential is required, as we want reflection to always gives us the same order when pulling for FieldInfo. My inspiration is from TLV Type-Length-Value. Let's have a look at the code:
public static byte[] StructToByteArray<T>(T obj)
{
using (MemoryStream ms = new MemoryStream())
{
FieldInfo[] infos = typeof(T).GetFields(BindingFlags.Public | BindingFlags.Instance);
foreach (FieldInfo info in infos)
{
BinaryFormatter bf = new BinaryFormatter();
using (MemoryStream inms = new MemoryStream()) {
bf.Serialize(inms, info.GetValue(obj));
byte[] ba = inms.ToArray();
// for length
ms.Write(BitConverter.GetBytes(ba.Length), 0, sizeof(int));
// for value
ms.Write(ba, 0, ba.Length);
}
}
return ms.ToArray();
}
}
The above function simply uses the BinaryFormatter to serialize the unknown size raw object, and I simply keep track of the size as well and store it inside the output MemoryStream too.
public static void ByteArrayToStruct<T>(byte[] data, out T output)
{
output = (T) Activator.CreateInstance(typeof(T), null);
using (MemoryStream ms = new MemoryStream(data))
{
byte[] ba = null;
FieldInfo[] infos = typeof(T).GetFields(BindingFlags.Public | BindingFlags.Instance);
foreach (FieldInfo info in infos)
{
// for length
ba = new byte[sizeof(int)];
ms.Read(ba, 0, sizeof(int));
// for value
int sz = BitConverter.ToInt32(ba, 0);
ba = new byte[sz];
ms.Read(ba, 0, sz);
BinaryFormatter bf = new BinaryFormatter();
using (MemoryStream inms = new MemoryStream(ba))
{
info.SetValue(output, bf.Deserialize(inms));
}
}
}
}
When we want to convert it back to its original struct we simply read the length back and directly dump it back into the BinaryFormatter which in turn dump it back into the struct.
These 2 functions are generic and should work with any struct, I've tested the above code in my C# project where I have a server and a client, connected and communicate via NamedPipeStream and I forward my struct as byte array from one and to another and converted it back.
I believe my approach might be better, since it doesn't fix length on the struct itself and the only overhead is just an int for every fields you have in your struct. There are also some tiny bit overhead inside the byte array generated by BinaryFormatter, but other than that, is not much.
I would take a look at the BinaryReader and BinaryWriter classes. I recently had to serialize data to a byte array (and back) and only found these classes after I'd basically rewritten them myself.
http://msdn.microsoft.com/en-us/library/system.io.binarywriter.aspx
There is a good example on that page too.
Looks like a predefined (C level) structure for some external library. Marshal is your friend. Check:
http://geekswithblogs.net/taylorrich/archive/2006/08/21/88665.aspx
for a starter how to deal with this. Note that you can - with attributes - define things like byte layout and string handling. VERY nice approach, actually.
Neither BinaryFormatter Nor MemoryStream are done for that.
#Abdel Olakara answer donese not work in .net 3.5, should be modified as below:
public static void ByteArrayToStructure<T>(byte[] bytearray, ref T obj)
{
int len = Marshal.SizeOf(obj);
IntPtr i = Marshal.AllocHGlobal(len);
Marshal.Copy(bytearray, 0, i, len);
obj = (T)Marshal.PtrToStructure(i, typeof(T));
Marshal.FreeHGlobal(i);
}
Header header = new Header();
Byte[] headerBytes = new Byte[Marshal.SizeOf(header)];
Marshal.Copy((IntPtr)(&header), headerBytes, 0, headerBytes.Length);
This should do the trick quickly, right?
This example here is only applicable to pure blittable types, e.g., types that can be memcpy'd directly in C.
Example - well known 64-bit struct
[StructLayout(LayoutKind.Sequential)]
public struct Voxel
{
public ushort m_id;
public byte m_red, m_green, m_blue, m_alpha, m_matid, m_custom;
}
Defined exactly like this, the struct will be automatically packed as 64-bit.
Now we can create volume of voxels:
Voxel[,,] voxels = new Voxel[16,16,16];
And save them all to a byte array:
int size = voxels.Length * 8; // Well known size: 64 bits
byte[] saved = new byte[size];
GCHandle h = GCHandle.Alloc(voxels, GCHandleType.Pinned);
Marshal.Copy(h.AddrOfPinnedObject(), saved, 0, size);
h.Free();
// now feel free to save 'saved' to a File / memory stream.
However, since the OP wants to know how to convert the struct itself, our Voxel struct can have following method ToBytes:
byte[] bytes = new byte[8]; // Well known size: 64 bits
GCHandle h = GCHandle.Alloc(this, GCHandleType.Pinned);
Marshal.Copy(hh.AddrOfPinnedObject(), bytes, 0, 8);
h.Free();
I want to do the equivalent of this:
byte[] byteArray;
enum commands : byte {one, two};
commands content = one;
byteArray = (byte*)&content;
yes, it's a byte now, but consider I want to change it in the future? how do I make byteArray contain content? (I don't care to copy it).
To convert any value types (not just primitive types) to byte arrays and vice versa:
public T FromByteArray<T>(byte[] rawValue)
{
GCHandle handle = GCHandle.Alloc(rawValue, GCHandleType.Pinned);
T structure = (T)Marshal.PtrToStructure(handle.AddrOfPinnedObject(), typeof(T));
handle.Free();
return structure;
}
public byte[] ToByteArray(object value, int maxLength)
{
int rawsize = Marshal.SizeOf(value);
byte[] rawdata = new byte[rawsize];
GCHandle handle =
GCHandle.Alloc(rawdata,
GCHandleType.Pinned);
Marshal.StructureToPtr(value,
handle.AddrOfPinnedObject(),
false);
handle.Free();
if (maxLength < rawdata.Length) {
byte[] temp = new byte[maxLength];
Array.Copy(rawdata, temp, maxLength);
return temp;
} else {
return rawdata;
}
}
The BitConverter class might be what you are looking for. Example:
int input = 123;
byte[] output = BitConverter.GetBytes(input);
If your enum was known to be an Int32 derived type, you could simply cast its values first:
BitConverter.GetBytes((int)commands.one);
You can use the BitConverter.GetBytes method to do this.
For anyone who's interested in how it works without using BitConverter you can do it like this:
// Convert double to byte[]
public unsafe byte[] pack(double d) {
byte[] packed = new byte[8]; // There are 8 bytes in a double
void* ptr = &d; // Get a reference to the memory containing the double
for (int i = 0; i < 8; i++) { // Each one of the 8 bytes needs to be added to the byte array
packed[i] = (byte)(*(UInt64 *)ptr >> (8 * i)); // Bit shift so that each chunk of 8 bits (1 byte) is cast as a byte and added to array
}
return packed;
}
// Convert byte[] to double
public unsafe double unpackDouble(byte[] data) {
double unpacked = 0.0; // Prepare a chunk of memory ready for the double
void* ptr = &unpacked; // Reference the double memory
for (int i = 0; i < data.Length; i++) {
*(UInt64 *)ptr |= ((UInt64)data[i] << (8 * i)); // Get the bits into the right place and OR into the double
}
return unpacked;
}
In reality it's much easier and safer to use BitConverter but it's fun to know!
I would like to get a byte[] from a float[] as quickly as possible, without looping through the whole array (via a cast, probably). Unsafe code is fine. Thanks!
I am looking for a byte array 4 time longer than the float array (the dimension of the byte array will be 4 times that of the float array, since each float is composed of 4 bytes). I'll pass this to a BinaryWriter.
EDIT:
To those critics screaming "premature optimization":
I have benchmarked this using ANTS profiler before I optimized. There was a significant speed increase because the file has a write-through cache and the float array is exactly sized to match the sector size on the disk. The binary writer wraps a file handle created with pinvoke'd win32 API. The optimization occurs since this lessens the number of function calls.
And, with regard to memory, this application creates massive caches which use plenty of memory. I can allocate the byte buffer once and re-use it many times--the double memory usage in this particular instance amounts to a roundoff error in the overall memory consumption of the app.
So I guess the lesson here is not to make premature assumptions ;)
There is a dirty fast (not unsafe code) way of doing this:
[StructLayout(LayoutKind.Explicit)]
struct BytetoDoubleConverter
{
[FieldOffset(0)]
public Byte[] Bytes;
[FieldOffset(0)]
public Double[] Doubles;
}
//...
static Double Sum(byte[] data)
{
BytetoDoubleConverter convert = new BytetoDoubleConverter { Bytes = data };
Double result = 0;
for (int i = 0; i < convert.Doubles.Length / sizeof(Double); i++)
{
result += convert.Doubles[i];
}
return result;
}
This will work, but I'm not sure of the support on Mono or newer versions of the CLR. The only strange thing is that the array.Length is the bytes length. This can be explained because it looks at the array length stored with the array, and because this array was a byte array that length will still be in byte length. The indexer does think about the Double being eight bytes large so no calculation is necessary there.
I've looked for it some more, and it's actually described on MSDN, How to: Create a C/C++ Union by Using Attributes (C# and Visual Basic), so chances are this will be supported in future versions. I am not sure about Mono though.
Premature optimization is the root of all evil! #Vlad's suggestion to iterate over each float is a much more reasonable answer than switching to a byte[]. Take the following table of runtimes for increasing numbers of elements (average of 50 runs):
Elements BinaryWriter(float) BinaryWriter(byte[])
-----------------------------------------------------------
10 8.72ms 8.76ms
100 8.94ms 8.82ms
1000 10.32ms 9.06ms
10000 32.56ms 10.34ms
100000 213.28ms 739.90ms
1000000 1955.92ms 10668.56ms
There is little difference between the two for small numbers of elements. Once you get into the huge number of elements range, the time spent copying from the float[] to the byte[] far outweighs the benefits.
So go with what is simple:
float[] data = new float[...];
foreach(float value in data)
{
writer.Write(value);
}
There is a way which avoids memory copying and iteration.
You can use a really ugly hack to temporary change your array to another type using (unsafe) memory manipulation.
I tested this hack in both 32 & 64 bit OS, so it should be portable.
The source + sample usage is maintained at https://gist.github.com/1050703 , but for your convenience I'll paste it here as well:
public static unsafe class FastArraySerializer
{
[StructLayout(LayoutKind.Explicit)]
private struct Union
{
[FieldOffset(0)] public byte[] bytes;
[FieldOffset(0)] public float[] floats;
}
[StructLayout(LayoutKind.Sequential, Pack = 1)]
private struct ArrayHeader
{
public UIntPtr type;
public UIntPtr length;
}
private static readonly UIntPtr BYTE_ARRAY_TYPE;
private static readonly UIntPtr FLOAT_ARRAY_TYPE;
static FastArraySerializer()
{
fixed (void* pBytes = new byte[1])
fixed (void* pFloats = new float[1])
{
BYTE_ARRAY_TYPE = getHeader(pBytes)->type;
FLOAT_ARRAY_TYPE = getHeader(pFloats)->type;
}
}
public static void AsByteArray(this float[] floats, Action<byte[]> action)
{
if (floats.handleNullOrEmptyArray(action))
return;
var union = new Union {floats = floats};
union.floats.toByteArray();
try
{
action(union.bytes);
}
finally
{
union.bytes.toFloatArray();
}
}
public static void AsFloatArray(this byte[] bytes, Action<float[]> action)
{
if (bytes.handleNullOrEmptyArray(action))
return;
var union = new Union {bytes = bytes};
union.bytes.toFloatArray();
try
{
action(union.floats);
}
finally
{
union.floats.toByteArray();
}
}
public static bool handleNullOrEmptyArray<TSrc,TDst>(this TSrc[] array, Action<TDst[]> action)
{
if (array == null)
{
action(null);
return true;
}
if (array.Length == 0)
{
action(new TDst[0]);
return true;
}
return false;
}
private static ArrayHeader* getHeader(void* pBytes)
{
return (ArrayHeader*)pBytes - 1;
}
private static void toFloatArray(this byte[] bytes)
{
fixed (void* pArray = bytes)
{
var pHeader = getHeader(pArray);
pHeader->type = FLOAT_ARRAY_TYPE;
pHeader->length = (UIntPtr)(bytes.Length / sizeof(float));
}
}
private static void toByteArray(this float[] floats)
{
fixed(void* pArray = floats)
{
var pHeader = getHeader(pArray);
pHeader->type = BYTE_ARRAY_TYPE;
pHeader->length = (UIntPtr)(floats.Length * sizeof(float));
}
}
}
And the usage is:
var floats = new float[] {0, 1, 0, 1};
floats.AsByteArray(bytes =>
{
foreach (var b in bytes)
{
Console.WriteLine(b);
}
});
If you do not want any conversion to happen, I would suggest Buffer.BlockCopy().
public static void BlockCopy(
Array src,
int srcOffset,
Array dst,
int dstOffset,
int count
)
For example:
float[] floatArray = new float[1000];
byte[] byteArray = new byte[floatArray.Length * 4];
Buffer.BlockCopy(floatArray, 0, byteArray, 0, byteArray.Length);
You're better-off letting the BinaryWriter do this for you. There's going to be iteration over your entire set of data regardless of which method you use, so there's no point in playing with bytes.
Although you can obtain a byte* pointer using unsafe and fixed, you cannot convert the byte* to byte[] in order for the writer to accept it as a parameter without performing data copy. Which you do not want to do as it will double your memory footprint and add an extra iteration over the inevitable iteration that needs to be performed in order to output the data to disk.
Instead, you are still better off iterating over the array of floats and writing each float to the writer individually, using the Write(double) method. It will still be fast because of buffering inside the writer. See sixlettervariables's numbers.
Using the new Span<> in .Net Core 2.1 or later...
byte[] byteArray2 = MemoryMarshal.Cast<float, byte>(floatArray).ToArray();
Or, if Span can be used instead, then a direct reinterpret cast can be done: (very fast - zero copying)
Span<byte> byteArray3 = MemoryMarshal.Cast<float, byte>(floatArray);
// with span we can get a byte, set a byte, iterate, and more.
byte someByte = byteSpan[2];
byteSpan[2] = 33;
I did some crude benchmarks. The time taken for each is in the comments. [release/no debugger/x64]
float[] floatArray = new float[100];
for (int i = 0; i < 100; i++) floatArray[i] = i * 7.7777f;
Stopwatch start = Stopwatch.StartNew();
for (int j = 0; j < 100; j++)
{
start.Restart();
for (int k = 0; k < 1000; k++)
{
Span<byte> byteSpan = MemoryMarshal.Cast<float, byte>(floatArray);
}
long timeTaken1 = start.ElapsedTicks; ////// 0 ticks //////
start.Restart();
for (int k = 0; k < 1000; k++)
{
byte[] byteArray2 = MemoryMarshal.Cast<float, byte>(floatArray).ToArray();
}
long timeTaken2 = start.ElapsedTicks; ////// 26 ticks //////
start.Restart();
for (int k = 0; k < 1000; k++)
{
byte[] byteArray = new byte[sizeof(float) * floatArray.Length];
for (int i = 0; i < floatArray.Length; i++)
BitConverter.GetBytes(floatArray[i]).CopyTo(byteArray, i * sizeof(float));
}
long timeTaken3 = start.ElapsedTicks; ////// 1310 ticks //////
start.Restart();
for (int k = 0; k < 1000; k++)
{
byte[] byteArray = new byte[sizeof(float) * floatArray.Length];
Buffer.BlockCopy(floatArray, 0, byteArray, 0, byteArray.Length);
}
long timeTaken4 = start.ElapsedTicks; ////// 33 ticks //////
start.Restart();
for (int k = 0; k < 1000; k++)
{
byte[] byteArray = new byte[sizeof(float) * floatArray.Length];
MemoryStream memStream = new MemoryStream();
BinaryWriter writer = new BinaryWriter(memStream);
foreach (float value in floatArray)
writer.Write(value);
writer.Close();
}
long timeTaken5 = start.ElapsedTicks; ////// 1080 ticks //////
Console.WriteLine($"{timeTaken1/10,6} {timeTaken2 / 10,6} {timeTaken3 / 10,6} {timeTaken4 / 10,6} {timeTaken5 / 10,6} ");
}
We have a class called LudicrousSpeedSerialization and it contains the following unsafe method:
static public byte[] ConvertFloatsToBytes(float[] data)
{
int n = data.Length;
byte[] ret = new byte[n * sizeof(float)];
if (n == 0) return ret;
unsafe
{
fixed (byte* pByteArray = &ret[0])
{
float* pFloatArray = (float*)pByteArray;
for (int i = 0; i < n; i++)
{
pFloatArray[i] = data[i];
}
}
}
return ret;
}
Although it basically does do a for loop behind the scenes, it does do the job in one line
byte[] byteArray = floatArray.Select(
f=>System.BitConverter.GetBytes(f)).Aggregate(
(bytes, f) => {List<byte> temp = bytes.ToList(); temp.AddRange(f); return temp.ToArray(); });
I use an extension method to convert float arrays into byte arrays:
public static unsafe byte[] ToByteArray(this float[] floatArray, int count)
{
int arrayLength = floatArray.Length > count ? count : floatArray.Length;
byte[] byteArray = new byte[4 * arrayLength];
fixed (float* floatPointer = floatArray)
{
fixed (byte* bytePointer = byteArray)
{
float* read = floatPointer;
float* write = (float*)bytePointer;
for (int i = 0; i < arrayLength; i++)
{
*write++ = *read++;
}
}
}
return byteArray;
}
I understand that an array is a pointer to memory associated with information on the type and number of elements. Also, it seems to me that there is no way of doing a conversion from and to a byte array without copying the data as above.
Have I understood this? Would it even be impossible to write IL to create an array from a pointer, type and length without copying data?
EDIT: Thanks for the answers, I learned some fundamentals and got to try out new tricks!
After initially accepting Davy Landman's answer I found out that while his brilliant StructLayout hack does convert byte arrays into float arrays, it does not work the other way around. To demonstrate:
[StructLayout(LayoutKind.Explicit)]
struct UnionArray
{
[FieldOffset(0)]
public Byte[] Bytes;
[FieldOffset(0)]
public float[] Floats;
}
static void Main(string[] args)
{
// From bytes to floats - works
byte[] bytes = { 0, 1, 2, 4, 8, 16, 32, 64 };
UnionArray arry = new UnionArray { Bytes = bytes };
for (int i = 0; i < arry.Bytes.Length / 4; i++)
Console.WriteLine(arry.Floats[i]);
// From floats to bytes - index out of range
float[] floats = { 0.1f, 0.2f, 0.3f };
arry = new UnionArray { Floats = floats };
for (int i = 0; i < arry.Floats.Length * 4; i++)
Console.WriteLine(arry.Bytes[i]);
}
It seems that the CLR sees both arrays as having the same length. If the struct is created from float data, the byte array's length is just too short.
You can use a really ugly hack to temporary change your array to byte[] using memory manipulation.
This is really fast and efficient as it doesn't require cloning the data and iterating on it.
I tested this hack in both 32 & 64 bit OS, so it should be portable.
The source + sample usage is maintained at https://gist.github.com/1050703 , but for your convenience I'll paste it here as well:
public static unsafe class FastArraySerializer
{
[StructLayout(LayoutKind.Explicit)]
private struct Union
{
[FieldOffset(0)] public byte[] bytes;
[FieldOffset(0)] public float[] floats;
}
[StructLayout(LayoutKind.Sequential, Pack = 1)]
private struct ArrayHeader
{
public UIntPtr type;
public UIntPtr length;
}
private static readonly UIntPtr BYTE_ARRAY_TYPE;
private static readonly UIntPtr FLOAT_ARRAY_TYPE;
static FastArraySerializer()
{
fixed (void* pBytes = new byte[1])
fixed (void* pFloats = new float[1])
{
BYTE_ARRAY_TYPE = getHeader(pBytes)->type;
FLOAT_ARRAY_TYPE = getHeader(pFloats)->type;
}
}
public static void AsByteArray(this float[] floats, Action<byte[]> action)
{
if (floats.handleNullOrEmptyArray(action))
return;
var union = new Union {floats = floats};
union.floats.toByteArray();
try
{
action(union.bytes);
}
finally
{
union.bytes.toFloatArray();
}
}
public static void AsFloatArray(this byte[] bytes, Action<float[]> action)
{
if (bytes.handleNullOrEmptyArray(action))
return;
var union = new Union {bytes = bytes};
union.bytes.toFloatArray();
try
{
action(union.floats);
}
finally
{
union.floats.toByteArray();
}
}
public static bool handleNullOrEmptyArray<TSrc,TDst>(this TSrc[] array, Action<TDst[]> action)
{
if (array == null)
{
action(null);
return true;
}
if (array.Length == 0)
{
action(new TDst[0]);
return true;
}
return false;
}
private static ArrayHeader* getHeader(void* pBytes)
{
return (ArrayHeader*)pBytes - 1;
}
private static void toFloatArray(this byte[] bytes)
{
fixed (void* pArray = bytes)
{
var pHeader = getHeader(pArray);
pHeader->type = FLOAT_ARRAY_TYPE;
pHeader->length = (UIntPtr)(bytes.Length / sizeof(float));
}
}
private static void toByteArray(this float[] floats)
{
fixed(void* pArray = floats)
{
var pHeader = getHeader(pArray);
pHeader->type = BYTE_ARRAY_TYPE;
pHeader->length = (UIntPtr)(floats.Length * sizeof(float));
}
}
}
And the usage is:
var floats = new float[] {0, 1, 0, 1};
floats.AsByteArray(bytes =>
{
foreach (var b in bytes)
{
Console.WriteLine(b);
}
});
Yes, the type information and data is in the same memory block, so that is impossible unless you overwrite the type information in a float array to fool the system that it's byte array. That would be a really ugly hack, and could easily blow up...
Here's how you can convert the floats without unsafe code if you like:
public static byte[] ToByteArray(this float[] floatArray) {
int len = floatArray.Length * 4;
byte[] byteArray = new byte[len];
int pos = 0;
foreach (float f in floatArray) {
byte[] data = BitConverter.GetBytes(f);
Array.Copy(data, 0, byteArray, pos, 4);
pos += 4;
}
return byteArray;
}
This question is the reverse of What is the fastest way to convert a float[] to a byte[]?.
I've answered with a union kind of hack to skip the whole copying of the data. You could easily reverse this (length = length *sizeof(Double).
I've written something similar for quick conversion between arrays. It's basically an ugly proof-of-concept more than a handsome solution. ;)
public static TDest[] ConvertArray<TSource, TDest>(TSource[] source)
where TSource : struct
where TDest : struct {
if (source == null)
throw new ArgumentNullException("source");
var sourceType = typeof(TSource);
var destType = typeof(TDest);
if (sourceType == typeof(char) || destType == typeof(char))
throw new NotSupportedException(
"Can not convert from/to a char array. Char is special " +
"in a somewhat unknown way (like enums can't be based on " +
"char either), and Marshal.SizeOf returns 1 even when the " +
"values held by a char can be above 255."
);
var sourceByteSize = Buffer.ByteLength(source);
var destTypeSize = Marshal.SizeOf(destType);
if (sourceByteSize % destTypeSize != 0)
throw new Exception(
"The source array is " + sourceByteSize + " bytes, which can " +
"not be transfered to chunks of " + destTypeSize + ", the size " +
"of type " + typeof(TDest).Name + ". Change destination type or " +
"pad the source array with additional values."
);
var destCount = sourceByteSize / destTypeSize;
var destArray = new TDest[destCount];
Buffer.BlockCopy(source, 0, destArray, 0, sourceByteSize);
return destArray;
}
}
public byte[] ToByteArray(object o)
{
int size = Marshal.SizeOf(o);
byte[] buffer = new byte[size];
IntPtr p = Marshal.AllocHGlobal(size);
try
{
Marshal.StructureToPtr(o, p, false);
Marshal.Copy(p, buffer, 0, size);
}
finally
{
Marshal.FreeHGlobal(p);
}
return buffer;
}
this may help you to convert an object to a byte array.
You should check my answer to a similar question: What is the fastest way to convert a float[] to a byte[]?.
In it you'll find portable code (32/64 bit compatible) to let you view a float array as a byte array or vice-versa, without copying the data. It's the fastest way that I know of to do such thing.
If you're just interested in the code, it's maintained at https://gist.github.com/1050703 .
Well - if you still interested in that hack - check out this modified code - it works like a charm and costs ~0 time, but it may not work in future since it's a hack allowing to gain full access to the whole process address space without trust requirements and unsafe marks.
[StructLayout(LayoutKind.Explicit)]
struct ArrayConvert
{
public static byte[] GetBytes(float[] floats)
{
ArrayConvert ar = new ArrayConvert();
ar.floats = floats;
ar.length.val = floats.Length * 4;
return ar.bytes;
}
public static float[] GetFloats(byte[] bytes)
{
ArrayConvert ar = new ArrayConvert();
ar.bytes = bytes;
ar.length.val = bytes.Length / 4;
return ar.floats;
}
public static byte[] GetTop4BytesFrom(object obj)
{
ArrayConvert ar = new ArrayConvert();
ar.obj = obj;
return new byte[]
{
ar.top4bytes.b0,
ar.top4bytes.b1,
ar.top4bytes.b2,
ar.top4bytes.b3
};
}
public static byte[] GetBytesFrom(object obj, int size)
{
ArrayConvert ar = new ArrayConvert();
ar.obj = obj;
ar.length.val = size;
return ar.bytes;
}
class ArrayLength
{
public int val;
}
class Top4Bytes
{
public byte b0;
public byte b1;
public byte b2;
public byte b3;
}
[FieldOffset(0)]
private Byte[] bytes;
[FieldOffset(0)]
private object obj;
[FieldOffset(0)]
private float[] floats;
[FieldOffset(0)]
private ArrayLength length;
[FieldOffset(0)]
private Top4Bytes top4bytes;
}