We Keep Coding

need help to make crc calculation's class thread safe

when we added paralllism elaboration on our application (dotnet service) we found some unexpected behavoir on crc calculation over text documents.
to isolate the issue i created a test case. the crc calculation fails when invoked from parallel looop. in this test case replacing parallel foreach with standard always fine. I think i've to made so change in crc32 class implementation, but i need some help to understand the right way. Thanks.
this the test method.
[TestMethod()]
public void Test_Crc_TestoDoc()
{
string query = #"select top 100 docId from sometable";
///key is document's id
///value is a couple, crc and text
Dictionary<int, Tuple<int, string>> docs = new Dictionary<int, Tuple<int, string>>();
using (SqlDataReader oSqlDataReader = Utility.ExecuteSP_Reader(query))
{
while (oSqlDataReader.Read())
{
int docId = oSqlDataReader.GetInt32(0);
///retrive the text by docId
string docText = Utility.GetDocText(docId);
///calculate and add crc in dic
int CRC = CRC32.Compute(docText);
docs.Add(docId, new Tuple<int, string>(CRC, docText));
}
oSqlDataReader.Close();
}
///calculate crc 100 times to check if the value
///is always the same for same text
for (int i = 0; i < 100; i++)
{
Parallel.ForEach(docs.Keys,(int docId) =>
{
///crc saved in dictionary
int CRC1 = docs[docId].Item1;
///text saved in dictionary
string docText = docs[docId].Item2;
///calculate crc again, crc2 must be equal to crc1 stored in dictionary
int CRC2 = CRC32.Compute(docText);
Assert.AreEqual(CRC1, CRC2, $"crc not equal, why? docId->{docId} CRC1->{CRC1} CRC2->{CRC2}");
});
}
}
crc32 class:
public class CRC32 : HashAlgorithm
{
#region CONSTRUCTORS
/// <summary>Creates a CRC32 object using the <see cref="DefaultPolynomial"/>.</summary>
public CRC32()
: this(DefaultPolynomial)
{
}
/// <summary>Creates a CRC32 object using the specified polynomial.</summary>
/// <remarks>The polynomical should be supplied in its bit-reflected form. <see cref="DefaultPolynomial"/>.</remarks>
[CLSCompliant(false)]
public CRC32(uint polynomial)
{
HashSizeValue = 32;
_crc32Table = (uint[])_crc32TablesCache[polynomial];
if (_crc32Table == null)
{
_crc32Table = CRC32._buildCRC32Table(polynomial);
_crc32TablesCache.Add(polynomial, _crc32Table);
}
Initialize();
}
// static constructor
static CRC32()
{
_crc32TablesCache = Hashtable.Synchronized(new Hashtable());
_defaultCRC = new CRC32();
}
#endregion
#region PROPERTIES
/// <summary>Gets the default polynomial (used in WinZip, Ethernet, etc.)</summary>
/// <remarks>The default polynomial is a bit-reflected version of the standard polynomial 0x04C11DB7 used by WinZip, Ethernet, etc.</remarks>
[CLSCompliant(false)]
public static readonly uint DefaultPolynomial = 0xEDB88320; // Bitwise reflection of 0x04C11DB7;
#endregion
#region METHODS
/// <summary>Initializes an implementation of HashAlgorithm.</summary>
public override void Initialize()
{
_crc = _allOnes;
}
/// <summary>Routes data written to the object into the hash algorithm for computing the hash.</summary>
protected override void HashCore(byte[] buffer, int offset, int count)
{
for (int i = offset; i < count; i++)
{
ulong ptr = (_crc & 0xFF) ^ buffer[i];
_crc >>= 8;
_crc ^= _crc32Table[ptr];
}
}
/// <summary>Finalizes the hash computation after the last data is processed by the cryptographic stream object.</summary>
protected override byte[] HashFinal()
{
byte[] finalHash = new byte[4];
ulong finalCRC = _crc ^ _allOnes;
finalHash[0] = (byte)((finalCRC >> 0) & 0xFF);
finalHash[1] = (byte)((finalCRC >> 8) & 0xFF);
finalHash[2] = (byte)((finalCRC >> 16) & 0xFF);
finalHash[3] = (byte)((finalCRC >> 24) & 0xFF);
return finalHash;
}
/// <summary>Computes the CRC32 value for the given ASCII string using the <see cref="DefaultPolynomial"/>.</summary>
public static int Compute(string asciiString)
{
_defaultCRC.Initialize();
return ToInt32(_defaultCRC.ComputeHash(asciiString));
}
/// <summary>Computes the CRC32 value for the given input stream using the <see cref="DefaultPolynomial"/>.</summary>
public static int Compute(Stream inputStream)
{
_defaultCRC.Initialize();
return ToInt32(_defaultCRC.ComputeHash(inputStream));
}
/// <summary>Computes the CRC32 value for the input data using the <see cref="DefaultPolynomial"/>.</summary>
public static int Compute(byte[] buffer)
{
_defaultCRC.Initialize();
return ToInt32(_defaultCRC.ComputeHash(buffer));
}
/// <summary>Computes the hash value for the input data using the <see cref="DefaultPolynomial"/>.</summary>
public static int Compute(byte[] buffer, int offset, int count)
{
_defaultCRC.Initialize();
return ToInt32(_defaultCRC.ComputeHash(buffer, offset, count));
}
/// <summary>Computes the hash value for the given ASCII string.</summary>
/// <remarks>The computation preserves the internal state between the calls, so it can be used for computation of a stream data.</remarks>
public byte[] ComputeHash(string asciiString)
{
byte[] rawBytes = ASCIIEncoding.ASCII.GetBytes(asciiString);
return ComputeHash(rawBytes);
}
/// <summary>Computes the hash value for the given input stream.</summary>
/// <remarks>The computation preserves the internal state between the calls, so it can be used for computation of a stream data.</remarks>
new public byte[] ComputeHash(Stream inputStream)
{
byte[] buffer = new byte[4096];
int bytesRead;
while ((bytesRead = inputStream.Read(buffer, 0, 4096)) > 0)
{
HashCore(buffer, 0, bytesRead);
}
return HashFinal();
}
/// <summary>Computes the hash value for the input data.</summary>
/// <remarks>The computation preserves the internal state between the calls, so it can be used for computation of a stream data.</remarks>
new public byte[] ComputeHash(byte[] buffer)
{
return ComputeHash(buffer, 0, buffer.Length);
}
/// <summary>Computes the hash value for the input data.</summary>
/// <remarks>The computation preserves the internal state between the calls, so it can be used for computation of a stream data.</remarks>
new public byte[] ComputeHash(byte[] buffer, int offset, int count)
{
HashCore(buffer, offset, count);
return HashFinal();
}
#endregion
#region PRIVATE SECTION
private static uint _allOnes = 0xffffffff;
private static CRC32 _defaultCRC;
private static Hashtable _crc32TablesCache;
private uint[] _crc32Table;
private uint _crc;
// Builds a crc32 table given a polynomial
private static uint[] _buildCRC32Table(uint polynomial)
{
uint crc;
uint[] table = new uint[256];
// 256 values representing ASCII character codes.
for (int i = 0; i < 256; i++)
{
crc = (uint)i;
for (int j = 8; j > 0; j--)
{
if ((crc & 1) == 1)
crc = (crc >> 1) ^ polynomial;
else
crc >>= 1;
}
table[i] = crc;
}
return table;
}
private static int ToInt32(byte[] buffer)
{
return BitConverter.ToInt32(buffer, 0);
}
#endregion
}

Probably the problem are all the "static" function.
In fact, a static function is the same for all of the instance of CRC32.
That means that while an instance is running, setting his parameter, another can write his own value over the first one.

Why can't I get a Byte[] from this DataRow?

I'm trying to get a byte[] from blob stored in SQLite, but the byte[] length is always 1
foreach (DataRow row in table.Rows)
{
Byte[] bytes = (Byte[])row["password_value"];
}
This the Wrapper Class I used, I suppose the problem in the wrapper I tested the code using normal SQLite wrapper it works fine, but I could not figure out what's the problem with wrapper
public class SQLiteBase
{
// imports system functions for work with pointers
[DllImport("kernel32")]
private extern static IntPtr HeapAlloc(IntPtr heap, UInt32 flags, UInt32 bytes);
[DllImport("kernel32")]
private extern static IntPtr GetProcessHeap();
[DllImport("kernel32")]
private extern static int lstrlen(IntPtr str);
// imports SQLite functions
[DllImport("sqlite3")]
private static extern int sqlite3_open(IntPtr fileName, out IntPtr database);
[DllImport("sqlite3")]
private static extern int sqlite3_close(IntPtr database);
[DllImport("sqlite3")]
private static extern int sqlite3_exec(IntPtr database, IntPtr query, IntPtr callback, IntPtr arguments,
out IntPtr error);
[DllImport("sqlite3")]
private static extern IntPtr sqlite3_errmsg(IntPtr database);
[DllImport("sqlite3")]
private static extern int sqlite3_prepare_v2(IntPtr database, IntPtr query, int length, out IntPtr statement,
out IntPtr tail);
[DllImport("sqlite3")]
private static extern int sqlite3_step(IntPtr statement);
[DllImport("sqlite3")]
private static extern int sqlite3_column_count(IntPtr statement);
[DllImport("sqlite3")]
private static extern IntPtr sqlite3_column_name(IntPtr statement, int columnNumber);
[DllImport("sqlite3")]
private static extern int sqlite3_column_type(IntPtr statement, int columnNumber);
[DllImport("sqlite3")]
private static extern int sqlite3_column_int(IntPtr statement, int columnNumber);
[DllImport("sqlite3")]
private static extern double sqlite3_column_double(IntPtr statement, int columnNumber);
[DllImport("sqlite3")]
private static extern IntPtr sqlite3_column_text(IntPtr statement, int columnNumber);
[DllImport("sqlite3")]
private static extern IntPtr sqlite3_column_blob(IntPtr statement, int columnNumber);
[DllImport("sqlite3")]
private static extern IntPtr sqlite3_column_table_name(IntPtr statement, int columnNumber);
[DllImport("sqlite3")]
private static extern int sqlite3_finalize(IntPtr handle);
// SQLite constants
private const int SQL_OK = 0;
private const int SQL_ROW = 100;
private const int SQL_DONE = 101;
/// <summary>
/// SQLite data types.
/// </summary>
public enum SQLiteDataTypes
{
/// <summary>
/// Integer numbers.
/// </summary>
INT = 1,
/// <summary>
/// Decimal numbers.
/// </summary>
FLOAT,
/// <summary>
/// All kinds of texts.
/// </summary>
TEXT,
/// <summary>
/// Blob objects - binary large objects.
/// </summary>
BLOB,
/// <summary>
/// Nothing.
/// </summary>
NULL
};
// pointer to database
private IntPtr database;
/// <summary>
/// Creates new instance of SQLiteBase class with no database attached.
/// </summary>
public SQLiteBase()
{
database = IntPtr.Zero;
}
/// <summary>
/// Creates new instance of SQLiteBase class and opens database with given name.
/// </summary>
/// <param name="baseName">Name (and path) to SQLite database file</param>
public SQLiteBase(String baseName)
{
OpenDatabase(baseName);
}
/// <summary>
/// Opens database.
/// </summary>
/// <param name="baseName">Name of database file</param>
public void OpenDatabase(String baseName)
{
// opens database
if (sqlite3_open(StringToPointer(baseName), out database) != SQL_OK)
{
// if there is some error, database pointer is set to 0 and exception is throws
database = IntPtr.Zero;
throw new Exception("Error with opening database " + baseName + "!");
}
}
/// <summary>
/// Closes opened database.
/// </summary>
public void CloseDatabase()
{
// closes the database if there is one opened
if (database != IntPtr.Zero)
{
sqlite3_close(database);
}
}
/// <summary>
/// Returns the list of tables in opened database.
/// </summary>
/// <returns></returns>
public ArrayList GetTables()
{
// executes query that select names of all tables and views in master table of every database
String query = "SELECT name FROM sqlite_master " +
"WHERE type IN ('table','view') AND name NOT LIKE 'sqlite_%'" +
"UNION ALL " +
"SELECT name FROM sqlite_temp_master " +
"WHERE type IN ('table','view') " +
"ORDER BY 1";
DataTable table = ExecuteQuery(query);
// when table is generater, it writes all table names in list that is returned
ArrayList list = new ArrayList();
foreach (DataRow row in table.Rows)
{
list.Add(row.ItemArray[0].ToString());
}
return list;
}
/// <summary>
/// Executes query that does not return anything (e.g. UPDATE, INSERT, DELETE).
/// </summary>
/// <param name="query"></param>
public void ExecuteNonQuery(String query)
{
// calles SQLite function that executes non-query
IntPtr error;
sqlite3_exec(database, StringToPointer(query), IntPtr.Zero, IntPtr.Zero, out error);
// if there is error, excetion is thrown
if (error != IntPtr.Zero)
throw new Exception("Error with executing non-query: \"" + query + "\"!\n" +
PointerToString(sqlite3_errmsg(error)));
}
/// <summary>
/// Executes query that does return something (e.g. SELECT).
/// </summary>
/// <param name="query"></param>
/// <returns></returns>
public DataTable ExecuteQuery(String query)
{
// processed query
IntPtr statement;
// excess data, it has no use
IntPtr excessData;
// process query and make statement
sqlite3_prepare_v2(database, StringToPointer(query), GetPointerLenght(StringToPointer(query)),
out statement, out excessData);
// table for result of function
DataTable table = new DataTable();
// reads first row - it is different from next rows because it also creates table columns
// result - returns SLQ_ROW while there is next row
int result = ReadFirstRow(statement, ref table);
// reads rows
while (result == SQL_ROW)
{
result = ReadNextRow(statement, ref table);
}
// finalize executing this query
sqlite3_finalize(statement);
// returns table
return table;
}
// private function for reading firs row and creating DataTable
private int ReadFirstRow(IntPtr statement, ref DataTable table)
{
// create new instance of DataTable with name "resultTable"
table = new DataTable("resultTable");
// evaluates statement
int resultType = sqlite3_step(statement);
// if result of statement is SQL_ROW, create new table and write row in it
if (resultType == SQL_ROW)
{
// returns number of columns returned by statement
int columnCount = sqlite3_column_count(statement);
// declartaion of variables for reading first row
String columnName = "";
int columnType = 0;
object[] columnValues = new object[columnCount];
// reads columns one by one
for (int i = 0; i < columnCount; i++)
{
// returns the name of current column
columnName = PointerToString(sqlite3_column_name(statement, i));
// returns the type of current column
columnType = sqlite3_column_type(statement, i);
// checks type of columns - neccessary because different functions are required for different types
switch (columnType)
{
// in case of integer column
case (int) SQLiteDataTypes.INT:
{
// adds new integer column to table
table.Columns.Add(columnName, Type.GetType("System.Int32"));
// writes column value in object array
columnValues[i] = sqlite3_column_int(statement, i);
break;
}
// same as for integer, this one is for float
case (int) SQLiteDataTypes.FLOAT:
{
table.Columns.Add(columnName, Type.GetType("System.Single"));
columnValues[i] = sqlite3_column_double(statement, i);
break;
}
// ... for text
case (int) SQLiteDataTypes.TEXT:
{
table.Columns.Add(columnName, Type.GetType("System.String"));
columnValues[i] = PointerToString(sqlite3_column_text(statement, i));
break;
}
// ... for blob - blob are written in table as strings!!
case (int) SQLiteDataTypes.BLOB:
{
table.Columns.Add(columnName, Type.GetType("System.Byte[]"));
columnValues[i] = PointerToByte(sqlite3_column_blob(statement, i));
break;
}
// in case of something other, value is read as string
default:
{
table.Columns.Add(columnName, Type.GetType("System.String"));
columnValues[i] = "";
break;
}
}
}
// writes column values to table
table.Rows.Add(columnValues);
}
// evalute statemnet for next results
return sqlite3_step(statement);
}
// private function for reading rows other than first
// it' same like first row, only without creating table and columns
private int ReadNextRow(IntPtr statement, ref DataTable table)
{
int columnCount = sqlite3_column_count(statement);
int columnType = 0;
object[] columnValues = new object[columnCount];
for (int i = 0; i < columnCount; i++)
{
columnType = sqlite3_column_type(statement, i);
switch (columnType)
{
case (int) SQLiteDataTypes.INT:
{
columnValues[i] = sqlite3_column_int(statement, i);
break;
}
case (int) SQLiteDataTypes.FLOAT:
{
columnValues[i] = sqlite3_column_double(statement, i);
break;
}
case (int) SQLiteDataTypes.TEXT:
{
columnValues[i] = PointerToString(sqlite3_column_text(statement, i));
break;
}
case (int) SQLiteDataTypes.BLOB:
{
columnValues[i] = PointerToByte(sqlite3_column_blob(statement, i));
break;
}
default:
{
columnValues[i] = "";
break;
}
}
}
table.Rows.Add(columnValues);
return sqlite3_step(statement);
}
// converts string to pointer
private IntPtr StringToPointer(String str)
{
// if string is null, pointer is 0
if (str == null)
{
return IntPtr.Zero;
}
else
{
// else, convert it to pointer
Encoding encoding = Encoding.UTF8;
Byte[] bytes = encoding.GetBytes(str);
int length = bytes.Length + 1;
IntPtr pointer = HeapAlloc(GetProcessHeap(), 0, (UInt32) length);
Marshal.Copy(bytes, 0, pointer, bytes.Length);
Marshal.WriteByte(pointer, bytes.Length, 0);
return pointer;
}
}
// convert pointer to string
private String PointerToString(IntPtr ptr)
{
if (ptr == IntPtr.Zero)
return null;
Encoding encoding = Encoding.UTF8;
int length = GetPointerLenght(ptr);
Byte[] bytes = new Byte[length];
Marshal.Copy(ptr, bytes, 0, length);
return encoding.GetString(bytes, 0, length);
}
private Byte[] PointerToByte(IntPtr ptr)
{
if (ptr == IntPtr.Zero)
return null;
int length = GetPointerLenght(ptr);
Byte[] bytes = new Byte[length];
Marshal.Copy(ptr, bytes, 0, length);
return bytes;
}
// returns length of pointer
private int GetPointerLenght(IntPtr ptr)
{
if (ptr == IntPtr.Zero)
return 0;
return lstrlen(ptr);
}
}
, I think the problem is in this section, or not
private Byte[] PointerToByte(IntPtr ptr)
{
if (ptr == IntPtr.Zero)
return null;
int length = GetPointerLenght(ptr);
Byte[] bytes = new Byte[length];
Marshal.Copy(ptr, bytes, 0, length);
return bytes;
}
UPDATE
private int GetPointerLenght(IntPtr ptr)
{
if (ptr == IntPtr.Zero)
return 0;
return lstrlen(ptr); // here the problem this gets the length of string and im passing an array of bytes
}
so the question now how to get the length of the byte array from pointer ?

Thanks to #FCin pointing out
sqlite3_column_bytes() or sqlite3_column_bytes16()
here the solution get the size or byte[] array from unmanaged code
[DllImport("sqlite3")]
private static extern int sqlite3_column_bytes(IntPtr statement, int columnNumber);
private int getSizeOfBytes(IntPtr statement,int i)
{
return sqlite3_column_bytes(statement, i);
}
ANd you pass it like that
columnValues[i] = PointerToByte(sqlite3_column_blob(statement, i), getSizeOfBytes(statement,i));

using pinvoke with structs and pointers

I am trying to pinvoke a c function within my c# code. It takes a struct and a double as an input and returns a struct of the same type. I've defined the struct the same in the c and c# code. when pinvoking the c function I get an exception "Methods type signature is not PInvoke compatible". Can somebody spot what I'm doing wrong? Thanks
C:
typedef struct myStruct_struct
{
double prefix[8];
int length;
double array[1];
}
myStruct;
extern "C" __declspec( dllexport ) myStruct *doSomething(const myStruct *inStruct, double val)
{
myStruct *outStruct;
//doSomething ...
return outStruct;
}
C#:
[StructLayout(LayoutKind.Sequential)]
public struct myStruct
{
[MarshalAs(UnmanagedType.ByValArray, SizeConst = 8)]
public double[] Prefix;
public int Length;
public IntPtr ArrayPtr;
public void MarshalArray(double[] array)
{
Length = array.Length;
int pointerSize = IntPtr.Size + (8 * Length);
ArrayPtr = Marshal.AllocHGlobal(Marshal.SizeOf(pointerSize));
Marshal.Copy(array, 0, ArrayPtr, Length);
}
public double[] UnMarshalArray()
{
double[] array = new double[Length];
Marshal.Copy(ArrayPtr, array, 0, Length);
return array;
}
}
[DllImport("testing.dll", EntryPoint = "doSomething", SetLastError = true, CharSet = CharSet.None, CallingConvention = CallingConvention.Cdecl)]
public static extern IntPtr test_doSomething(IntPtr inStruct, double val);
private void button1_Click(object sender, EventArgs e)
{
try
{
myStruct s = createStruct();
myStruct result = MarshalIn(test_doSomething(MarshalOut(s), 2));
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
}
private myStruct MarshalIn(IntPtr intPtr)
{
myStruct s = (myStruct)Marshal.PtrToStructure(intPtr, typeof(myStruct));
s.UnMarshalArray();
return s;
}
private IntPtr MarshalOut(myStruct s)
{
double[] array = new double[] { 1, 1, 2, 2, 1, 1, 2, 2 };
s.MarshalArray(array);
IntPtr outPtr = Marshal.AllocHGlobal(Marshal.SizeOf(typeof(myStruct)));
Marshal.StructureToPtr(s, outPtr, true);
return outPtr;
}

I understand this might not be the best solution in the world, but it is working for me in my tests, so I will leave you to infer things that you may personally need to do with regard to memory releasing, and leave you to your own judgment regarding that.
I personally HATE using Marshal.Copy and Buffer.BlockCopy. I wrote my own line of functions for this, here: http://nmcsmem.codeplex.com/ (Specifically the DTCSMemory project -> MemPtr structure), but I programmed this with the standard functions for easy portability. I have to admit, I may be doing some funky things with the bytes there, but I was just making doubly (no pun intended) sure.
IntPtr.Size has to be used if the structure is packed with normal packing. Normally, a simple structure interop would take care of that, but we are not using simply structure interop, here. If the structure is byte-packed, then you'll want to change the places where it says 'IntPtr.Size' back to just '4'.
That said, provided that we have this in C:
typedef struct myStruct_struct
{
int length;
double array[1];
}
myStruct;
__declspec(dllexport) myStruct *doSomething(const myStruct *inStruct, double val)
{
int i = sizeof(double);
//doSomething ...
myStruct *outStruct = (myStruct*)GlobalAlloc(0, sizeof(void*) + (8 * 256));
ZeroMemory(outStruct, sizeof(void*) + (8 * 256));
outStruct->length = 256;
outStruct->array[0] = inStruct->array[0] + val;
return outStruct;
}
Then this code in C# would work:
public class Program
{
/// <summary>
/// myStruct is not marshaled, directly.
/// </summary>
public struct myStruct
{
public int Length;
public double[] Array;
private IntPtr _ptr;
/// <summary>
/// Custom marshal a structure in from interop (and optionally free the original pointer).
/// </summary>
/// <param name="ptr"></param>
/// <param name="freeOrig"></param>
/// <returns></returns>
public static myStruct MarshalIn(IntPtr ptr, bool freeOrig = true)
{
byte[] by = new byte[4];
myStruct ns = new myStruct();
Marshal.Copy(ptr, by, 0, 4);
ns.Length = BitConverter.ToInt32(by, 0);
ns.Array = new double[ns.Length];
by = new byte[ns.Length * 8];
Marshal.Copy(ptr + IntPtr.Size, by, 0, by.Length);
Buffer.BlockCopy(by, 0, ns.Array, 0, by.Length);
if (freeOrig) Marshal.FreeHGlobal(ptr);
return ns;
}
/// <summary>
/// Custom marshal a structure for calling interop.
/// </summary>
/// <returns></returns>
public IntPtr MarshalOut()
{
IntPtr ptr;
int l = IntPtr.Size + (8 * Array.Length);
ptr = Marshal.AllocHGlobal(l);
byte[] by = BitConverter.GetBytes(Length);
Marshal.Copy(by, 0, ptr, 4);
by = new byte[Length * 8];
Buffer.BlockCopy(Array, 0, by, 0, by.Length);
Marshal.Copy(by, 0, ptr + IntPtr.Size, by.Length);
_ptr = ptr;
return ptr;
}
/// <summary>
/// Free any associated pointer with this structure created with MarshalOut().
/// </summary>
public void Free()
{
if (_ptr != IntPtr.Zero)
{
Marshal.FreeHGlobal(_ptr);
_ptr = IntPtr.Zero;
}
}
}
[DllImport("mylib.dll", SetLastError = true, CharSet = CharSet.None, CallingConvention = CallingConvention.Cdecl)]
public static extern IntPtr doSomething(IntPtr inStruct, double val);
static void Main()
{
// let's do some math.
myStruct ms = new myStruct(), ms2;
ms.Array = new double[1];
ms.Length = 1;
ms.Array[0] = 424.444;
ms2 = myStruct.MarshalIn(doSomething(ms.MarshalOut(), 524.444));
// Free this after the call.
ms.Free();
}
}

C# performance - Using unsafe pointers instead of IntPtr and Marshal

Question
I'm porting a C application into C#. The C app calls lots of functions from a 3rd-party DLL, so I wrote P/Invoke wrappers for these functions in C#. Some of these C functions allocate data which I have to use in the C# app, so I used IntPtr's, Marshal.PtrToStructure and Marshal.Copy to copy the native data (arrays and structures) into managed variables.
Unfortunately, the C# app proved to be much slower than the C version. A quick performance analysis showed that the above mentioned marshaling-based data copying is the bottleneck. I'm considering to speed up the C# code by rewriting it to use pointers instead. Since I don't have experience with unsafe code and pointers in C#, I need expert opinion regarding the following questions:
What are the drawbacks of using unsafe code and pointers instead of IntPtr and Marshaling? For example, is it more unsafe (pun intended) in any way? People seem to prefer marshaling, but I don't know why.
Is using pointers for P/Invoking really faster than using marshaling? How much speedup can be expected approximately? I couldn't find any benchmark tests for this.
Example code
To make the situation more clear, I hacked together a small example code (the real code is much more complex). I hope this example shows what I mean when I'm talking about "unsafe code and pointers" vs. "IntPtr and Marshal".
C library (DLL)
MyLib.h
#ifndef _MY_LIB_H_
#define _MY_LIB_H_
struct MyData
{
int length;
unsigned char* bytes;
};
__declspec(dllexport) void CreateMyData(struct MyData** myData, int length);
__declspec(dllexport) void DestroyMyData(struct MyData* myData);
#endif // _MY_LIB_H_
MyLib.c
#include <stdlib.h>
#include "MyLib.h"
void CreateMyData(struct MyData** myData, int length)
{
int i;
*myData = (struct MyData*)malloc(sizeof(struct MyData));
if (*myData != NULL)
{
(*myData)->length = length;
(*myData)->bytes = (unsigned char*)malloc(length * sizeof(char));
if ((*myData)->bytes != NULL)
for (i = 0; i < length; ++i)
(*myData)->bytes[i] = (unsigned char)(i % 256);
}
}
void DestroyMyData(struct MyData* myData)
{
if (myData != NULL)
{
if (myData->bytes != NULL)
free(myData->bytes);
free(myData);
}
}
C application
Main.c
#include <stdio.h>
#include "MyLib.h"
void main()
{
struct MyData* myData = NULL;
int length = 100 * 1024 * 1024;
printf("=== C++ test ===\n");
CreateMyData(&myData, length);
if (myData != NULL)
{
printf("Length: %d\n", myData->length);
if (myData->bytes != NULL)
printf("First: %d, last: %d\n", myData->bytes[0], myData->bytes[myData->length - 1]);
else
printf("myData->bytes is NULL");
}
else
printf("myData is NULL\n");
DestroyMyData(myData);
getchar();
}
C# application, which uses IntPtr and Marshal
Program.cs
using System;
using System.Runtime.InteropServices;
public static class Program
{
[StructLayout(LayoutKind.Sequential)]
private struct MyData
{
public int Length;
public IntPtr Bytes;
}
[DllImport("MyLib.dll")]
private static extern void CreateMyData(out IntPtr myData, int length);
[DllImport("MyLib.dll")]
private static extern void DestroyMyData(IntPtr myData);
public static void Main()
{
Console.WriteLine("=== C# test, using IntPtr and Marshal ===");
int length = 100 * 1024 * 1024;
IntPtr myData1;
CreateMyData(out myData1, length);
if (myData1 != IntPtr.Zero)
{
MyData myData2 = (MyData)Marshal.PtrToStructure(myData1, typeof(MyData));
Console.WriteLine("Length: {0}", myData2.Length);
if (myData2.Bytes != IntPtr.Zero)
{
byte[] bytes = new byte[myData2.Length];
Marshal.Copy(myData2.Bytes, bytes, 0, myData2.Length);
Console.WriteLine("First: {0}, last: {1}", bytes[0], bytes[myData2.Length - 1]);
}
else
Console.WriteLine("myData.Bytes is IntPtr.Zero");
}
else
Console.WriteLine("myData is IntPtr.Zero");
DestroyMyData(myData1);
Console.ReadKey(true);
}
}
C# application, which uses unsafe code and pointers
Program.cs
using System;
using System.Runtime.InteropServices;
public static class Program
{
[StructLayout(LayoutKind.Sequential)]
private unsafe struct MyData
{
public int Length;
public byte* Bytes;
}
[DllImport("MyLib.dll")]
private unsafe static extern void CreateMyData(out MyData* myData, int length);
[DllImport("MyLib.dll")]
private unsafe static extern void DestroyMyData(MyData* myData);
public unsafe static void Main()
{
Console.WriteLine("=== C# test, using unsafe code ===");
int length = 100 * 1024 * 1024;
MyData* myData;
CreateMyData(out myData, length);
if (myData != null)
{
Console.WriteLine("Length: {0}", myData->Length);
if (myData->Bytes != null)
Console.WriteLine("First: {0}, last: {1}", myData->Bytes[0], myData->Bytes[myData->Length - 1]);
else
Console.WriteLine("myData.Bytes is null");
}
else
Console.WriteLine("myData is null");
DestroyMyData(myData);
Console.ReadKey(true);
}
}

It's a little old thread, but I recently made excessive performance tests with marshaling in C#. I need to unmarshal lots of data from a serial port over many days. It was important to me to have no memory leaks (because the smallest leak will get significant after a couple of million calls) and I also made a lot of statistical performance (time used) tests with very big structs (>10kb) just for the sake of it (an no, you should never have a 10kb struct :-) )
I tested the following three unmarshalling strategies (I also tested the marshalling). In nearly all cases the first one (MarshalMatters) outperformed the other two.
Marshal.Copy was always slowest by far, the other two were mostly very close together in the race.
Using unsafe code can pose a significant security risk.
First:
public class MarshalMatters
{
public static T ReadUsingMarshalUnsafe<T>(byte[] data) where T : struct
{
unsafe
{
fixed (byte* p = &data[0])
{
return (T)Marshal.PtrToStructure(new IntPtr(p), typeof(T));
}
}
}
public unsafe static byte[] WriteUsingMarshalUnsafe<selectedT>(selectedT structure) where selectedT : struct
{
byte[] byteArray = new byte[Marshal.SizeOf(structure)];
fixed (byte* byteArrayPtr = byteArray)
{
Marshal.StructureToPtr(structure, (IntPtr)byteArrayPtr, true);
}
return byteArray;
}
}
Second:
public class Adam_Robinson
{
private static T BytesToStruct<T>(byte[] rawData) where T : struct
{
T result = default(T);
GCHandle handle = GCHandle.Alloc(rawData, GCHandleType.Pinned);
try
{
IntPtr rawDataPtr = handle.AddrOfPinnedObject();
result = (T)Marshal.PtrToStructure(rawDataPtr, typeof(T));
}
finally
{
handle.Free();
}
return result;
}
/// <summary>
/// no Copy. no unsafe. Gets a GCHandle to the memory via Alloc
/// </summary>
/// <typeparam name="selectedT"></typeparam>
/// <param name="structure"></param>
/// <returns></returns>
public static byte[] StructToBytes<T>(T structure) where T : struct
{
int size = Marshal.SizeOf(structure);
byte[] rawData = new byte[size];
GCHandle handle = GCHandle.Alloc(rawData, GCHandleType.Pinned);
try
{
IntPtr rawDataPtr = handle.AddrOfPinnedObject();
Marshal.StructureToPtr(structure, rawDataPtr, false);
}
finally
{
handle.Free();
}
return rawData;
}
}
Third:
/// <summary>
/// http://stackoverflow.com/questions/2623761/marshal-ptrtostructure-and-back-again-and-generic-solution-for-endianness-swap
/// </summary>
public class DanB
{
/// <summary>
/// uses Marshal.Copy! Not run in unsafe. Uses AllocHGlobal to get new memory and copies.
/// </summary>
public static byte[] GetBytes<T>(T structure) where T : struct
{
var size = Marshal.SizeOf(structure); //or Marshal.SizeOf<selectedT>(); in .net 4.5.1
byte[] rawData = new byte[size];
IntPtr ptr = Marshal.AllocHGlobal(size);
Marshal.StructureToPtr(structure, ptr, true);
Marshal.Copy(ptr, rawData, 0, size);
Marshal.FreeHGlobal(ptr);
return rawData;
}
public static T FromBytes<T>(byte[] bytes) where T : struct
{
var structure = new T();
int size = Marshal.SizeOf(structure); //or Marshal.SizeOf<selectedT>(); in .net 4.5.1
IntPtr ptr = Marshal.AllocHGlobal(size);
Marshal.Copy(bytes, 0, ptr, size);
structure = (T)Marshal.PtrToStructure(ptr, structure.GetType());
Marshal.FreeHGlobal(ptr);
return structure;
}
}

Considerations in Interoperability explains why and when Marshaling is required and at what cost. Quote:
Marshaling occurs when a caller and a callee cannot operate on the same instance of data.
repeated marshaling can negatively affect the performance of your application.
Therefore, answering your question if
... using pointers for P/Invoking really faster than using marshaling ...
first ask yourself a question if the managed code is able to operate on the unmanaged method return value instance. If the answer is yes then Marshaling and the associated performance cost is not required.
The approximate time saving would be O(n) function where n of the size of the marshalled instance.
In addition, not keeping both managed and unmanaged blocks of data in memory at the same time for the duration of the method (in "IntPtr and Marshal" example) eliminates additional overhead and the memory pressure.
What are the drawbacks of using unsafe code and pointers ...
The drawback is the risk associated with accessing the memory directly through pointers. There is nothing less safe to it than using pointers in C or C++. Use it if needed and makes sense. More details are here.
There is one "safety" concern with the presented examples: releasing of allocated unmanaged memory is not guaranteed after the managed code errors. The best practice is to
CreateMyData(out myData1, length);
if(myData1!=IntPtr.Zero) {
try {
// -> use myData1
...
// <-
}
finally {
DestroyMyData(myData1);
}
}

For anyone still reading,
Something I don't think I saw in any of the answers, - unsafe code does present something of a security risk. It's not a huge risk, it would be something quite challenging to exploit. However, if like me you work in a PCI compliant organization, unsafe code is disallowed by policy for this reason.
Managed code is normally very secure because the CLR takes care of memory location and allocation, preventing you from accessing or writing any memory you're not supposed to.
When you use the unsafe keyword and compile with '/unsafe' and use pointers, you bypass these checks and create the potential for someone to use your application to gain some level of unauthorized access to the machine it is running on. Using something like a buffer-overrun attack, your code could be tricked into writing instructions into an area of memory that might then be accessed by the program counter (i.e. code injection), or just crash the machine.
Many years ago, SQL server actually fell prey to malicious code delivered in a TDS packet that was far longer than it was supposed to be. The method reading the packet didn't check the length and continued to write the contents past the reserved address space. The extra length and content were carefully crafted such that it wrote an entire program into memory - at the address of the next method.
The attacker then had their own code being executed by the SQL server within a context that had the highest level of access. It didn't even need to break the encryption as the vulnerability was below this point in the transport layer stack.

Just wanted to add my experience to this old thread:
We used Marshaling in sound recording software - we received real time sound data from mixer into native buffers and marshaled it to byte[]. That was real performance killer. We were forced to move to unsafe structs as the only way to complete the task.
In case you don't have large native structs and don't mind that all data is filled twice - Marshaling is more elegant and much, much safer approach.

Two answers,
Unsafe code means it is not managed by the CLR. You need to take care of resources it uses.
You cannot scale the performance because there are so many factors effecting it. But definitely using pointers will be much faster.

Because you stated that your code calls to 3rd-party DLL, I think the unsafe code is more suited in you scenario. You ran into a particular situation of wapping variable-length array in a struct; I know, I know this kind of usage occurs all the time, but it's not always the case after all. You might want to have a look of some questions about this, for example:
How do I marshal a struct that contains a variable-sized array to C#?
If .. I say if .. you can modify the third party libraries a bit for this particular case, then you might consider the following usage:
using System.Runtime.InteropServices;
public static class Program { /*
[StructLayout(LayoutKind.Sequential)]
private struct MyData {
public int Length;
public byte[] Bytes;
} */
[DllImport("MyLib.dll")]
// __declspec(dllexport) void WINAPI CreateMyDataAlt(BYTE bytes[], int length);
private static extern void CreateMyDataAlt(byte[] myData, ref int length);
/*
[DllImport("MyLib.dll")]
private static extern void DestroyMyData(byte[] myData); */
public static void Main() {
Console.WriteLine("=== C# test, using IntPtr and Marshal ===");
int length = 100*1024*1024;
var myData1 = new byte[length];
CreateMyDataAlt(myData1, ref length);
if(0!=length) {
// MyData myData2 = (MyData)Marshal.PtrToStructure(myData1, typeof(MyData));
Console.WriteLine("Length: {0}", length);
/*
if(myData2.Bytes!=IntPtr.Zero) {
byte[] bytes = new byte[myData2.Length];
Marshal.Copy(myData2.Bytes, bytes, 0, myData2.Length); */
Console.WriteLine("First: {0}, last: {1}", myData1[0], myData1[length-1]); /*
}
else {
Console.WriteLine("myData.Bytes is IntPtr.Zero");
} */
}
else {
Console.WriteLine("myData is empty");
}
// DestroyMyData(myData1);
Console.ReadKey(true);
}
}
As you can see much of your original marshalling code is commented out, and declared a CreateMyDataAlt(byte[], ref int) for a coresponding modified external unmanaged function CreateMyDataAlt(BYTE [], int). Some of the data copy and pointer check turns to be unnecessary, that says, the code can be even simpler and probably runs faster.
So, what's so different with the modification? The byte array is now marshalled directly without warpping in a struct and passed to the unmanaged side. You don't allocate the memory within the unmanaged code, rather, just filling data to it(implementation details omitted); and after the call, the data needed is provided to the managed side. If you want to present that the data is not filled and should not be used, you can simply set length to zero to tell the managed side. Because the byte array is allocated within the managed side, it'll be collected sometime, you don't have to take care of that.

I had the same question today and I was looking for some concrete measurement values, but I couldn't find any. So I wrote my own tests.
The test is copying pixel data of a 10k x 10k RGB image. The image data is 300 MB (3*10^9 bytes). Some methods copy this data 10 times, others are faster and therefore copy it 100 times. The used copying methods include
array access via byte pointer
Marshal.Copy(): a) 1 * 300 MB, b) 1e9 * 3 bytes
Buffer.BlockCopy(): a) 1 * 300 MB, b) 1e9 * 3 bytes
Test environment:
CPU: Intel Core i7-3630QM # 2.40 GHz
OS: Win 7 Pro x64 SP1
Visual Studio 2015.3, code is C++/CLI, targeted .net version is 4.5.2, compiled for Debug.
Test results:
The CPU load is 100% for 1 core at all methods (equals 12.5% total CPU load).
Comparison of speed and execution time:
method speed exec.time
Marshal.Copy (1*300MB) 100 % 100%
Buffer.BlockCopy (1*300MB) 98 % 102%
Pointer 4.4 % 2280%
Buffer.BlockCopy (1e9*3B) 1.4 % 7120%
Marshal.Copy (1e9*3B) 0.95% 10600%
Execution times and calculated average throughput written as comments in the code below.
//------------------------------------------------------------------------------
static void CopyIntoBitmap_Pointer (array<unsigned char>^ i_aui8ImageData,
BitmapData^ i_ptrBitmap,
int i_iBytesPerPixel)
{
char* scan0 = (char*)(i_ptrBitmap->Scan0.ToPointer ());
int ixCnt = 0;
for (int ixRow = 0; ixRow < i_ptrBitmap->Height; ixRow++)
{
for (int ixCol = 0; ixCol < i_ptrBitmap->Width; ixCol++)
{
char* pPixel = scan0 + ixRow * i_ptrBitmap->Stride + ixCol * 3;
pPixel[0] = i_aui8ImageData[ixCnt++];
pPixel[1] = i_aui8ImageData[ixCnt++];
pPixel[2] = i_aui8ImageData[ixCnt++];
}
}
}
//------------------------------------------------------------------------------
static void CopyIntoBitmap_MarshallLarge (array<unsigned char>^ i_aui8ImageData,
BitmapData^ i_ptrBitmap)
{
IntPtr ptrScan0 = i_ptrBitmap->Scan0;
Marshal::Copy (i_aui8ImageData, 0, ptrScan0, i_aui8ImageData->Length);
}
//------------------------------------------------------------------------------
static void CopyIntoBitmap_MarshalSmall (array<unsigned char>^ i_aui8ImageData,
BitmapData^ i_ptrBitmap,
int i_iBytesPerPixel)
{
int ixCnt = 0;
for (int ixRow = 0; ixRow < i_ptrBitmap->Height; ixRow++)
{
for (int ixCol = 0; ixCol < i_ptrBitmap->Width; ixCol++)
{
IntPtr ptrScan0 = IntPtr::Add (i_ptrBitmap->Scan0, i_iBytesPerPixel);
Marshal::Copy (i_aui8ImageData, ixCnt, ptrScan0, i_iBytesPerPixel);
ixCnt += i_iBytesPerPixel;
}
}
}
//------------------------------------------------------------------------------
void main ()
{
int iWidth = 10000;
int iHeight = 10000;
int iBytesPerPixel = 3;
Bitmap^ oBitmap = gcnew Bitmap (iWidth, iHeight, PixelFormat::Format24bppRgb);
BitmapData^ oBitmapData = oBitmap->LockBits (Rectangle (0, 0, iWidth, iHeight), ImageLockMode::WriteOnly, oBitmap->PixelFormat);
array<unsigned char>^ aui8ImageData = gcnew array<unsigned char> (iWidth * iHeight * iBytesPerPixel);
int ixCnt = 0;
for (int ixRow = 0; ixRow < iHeight; ixRow++)
{
for (int ixCol = 0; ixCol < iWidth; ixCol++)
{
aui8ImageData[ixCnt++] = ixRow * 250 / iHeight;
aui8ImageData[ixCnt++] = ixCol * 250 / iWidth;
aui8ImageData[ixCnt++] = ixCol;
}
}
//========== Pointer ==========
// ~ 8.97 sec for 10k * 10k * 3 * 10 exec, ~ 334 MB/s
int iExec = 10;
DateTime dtStart = DateTime::Now;
for (int ixExec = 0; ixExec < iExec; ixExec++)
{
CopyIntoBitmap_Pointer (aui8ImageData, oBitmapData, iBytesPerPixel);
}
TimeSpan tsDuration = DateTime::Now - dtStart;
Console::WriteLine (tsDuration + " " + ((double)aui8ImageData->Length * iExec / tsDuration.TotalSeconds / 1e6));
//========== Marshal.Copy, 1 large block ==========
// 3.94 sec for 10k * 10k * 3 * 100 exec, ~ 7617 MB/s
iExec = 100;
dtStart = DateTime::Now;
for (int ixExec = 0; ixExec < iExec; ixExec++)
{
CopyIntoBitmap_MarshallLarge (aui8ImageData, oBitmapData);
}
tsDuration = DateTime::Now - dtStart;
Console::WriteLine (tsDuration + " " + ((double)aui8ImageData->Length * iExec / tsDuration.TotalSeconds / 1e6));
//========== Marshal.Copy, many small 3-byte blocks ==========
// 41.7 sec for 10k * 10k * 3 * 10 exec, ~ 72 MB/s
iExec = 10;
dtStart = DateTime::Now;
for (int ixExec = 0; ixExec < iExec; ixExec++)
{
CopyIntoBitmap_MarshalSmall (aui8ImageData, oBitmapData, iBytesPerPixel);
}
tsDuration = DateTime::Now - dtStart;
Console::WriteLine (tsDuration + " " + ((double)aui8ImageData->Length * iExec / tsDuration.TotalSeconds / 1e6));
//========== Buffer.BlockCopy, 1 large block ==========
// 4.02 sec for 10k * 10k * 3 * 100 exec, ~ 7467 MB/s
iExec = 100;
array<unsigned char>^ aui8Buffer = gcnew array<unsigned char> (aui8ImageData->Length);
dtStart = DateTime::Now;
for (int ixExec = 0; ixExec < iExec; ixExec++)
{
Buffer::BlockCopy (aui8ImageData, 0, aui8Buffer, 0, aui8ImageData->Length);
}
tsDuration = DateTime::Now - dtStart;
Console::WriteLine (tsDuration + " " + ((double)aui8ImageData->Length * iExec / tsDuration.TotalSeconds / 1e6));
//========== Buffer.BlockCopy, many small 3-byte blocks ==========
// 28.0 sec for 10k * 10k * 3 * 10 exec, ~ 107 MB/s
iExec = 10;
dtStart = DateTime::Now;
for (int ixExec = 0; ixExec < iExec; ixExec++)
{
int ixCnt = 0;
for (int ixRow = 0; ixRow < iHeight; ixRow++)
{
for (int ixCol = 0; ixCol < iWidth; ixCol++)
{
Buffer::BlockCopy (aui8ImageData, ixCnt, aui8Buffer, ixCnt, iBytesPerPixel);
ixCnt += iBytesPerPixel;
}
}
}
tsDuration = DateTime::Now - dtStart;
Console::WriteLine (tsDuration + " " + ((double)aui8ImageData->Length * iExec / tsDuration.TotalSeconds / 1e6));
oBitmap->UnlockBits (oBitmapData);
oBitmap->Save ("d:\\temp\\bitmap.bmp", ImageFormat::Bmp);
}
related information:
Why is memcpy() and memmove() faster than pointer increments?
Array.Copy vs Buffer.BlockCopy, Answer https://stackoverflow.com/a/33865267
https://github.com/dotnet/coreclr/issues/2430 "Array.Copy & Buffer.BlockCopy x2 to x3 slower < 1kB"
https://github.com/dotnet/coreclr/blob/master/src/vm/comutilnative.cpp, Line 718 at the time of writing: Buffer.BlockCopy() uses memmove

Using UpdateResource in C#?

I'm trying to change the icon of external executable programmatically. I've googled and found much information about this problem using C++. Basically, I need to use BeginUpdateResource, UpdateResource and EndUpdateResource. The problem is - I don't know what to pass to UpdateResource in C#.
Here's the code I have so far:
class IconChanger
{
[DllImport("kernel32.dll", SetLastError = true)]
static extern IntPtr BeginUpdateResource(string pFileName,
[MarshalAs(UnmanagedType.Bool)]bool bDeleteExistingResources);
[DllImport("kernel32.dll", SetLastError = true)]
static extern bool UpdateResource(IntPtr hUpdate, string lpType, string lpName, ushort wLanguage,
IntPtr lpData, uint cbData);
[DllImport("kernel32.dll", SetLastError = true)]
static extern bool EndUpdateResource(IntPtr hUpdate, bool fDiscard);
public enum ICResult
{
Success,
FailBegin,
FailUpdate,
FailEnd
}
public ICResult ChangeIcon(string exeFilePath, byte[] iconData)
{
// Load executable
IntPtr handleExe = BeginUpdateResource(exeFilePath, false);
if (handleExe == null)
return ICResult.FailBegin;
// Get language identifier
CultureInfo currentCulture = CultureInfo.CurrentCulture;
int pid = ((ushort)currentCulture.LCID) & 0x3ff;
int sid = ((ushort)currentCulture.LCID) >> 10;
ushort languageID = (ushort)((((ushort)pid) << 10) | ((ushort)sid));
// Get pointer to data
GCHandle iconHandle = GCHandle.Alloc(iconData, GCHandleType.Pinned);
// Replace the icon
if (UpdateResource(handleExe, "#3", "#1", languageID, iconHandle.AddrOfPinnedObject(), (uint)iconData.Length))
{
if (EndUpdateResource(handleExe, false))
return ICResult.Success;
else
return ICResult.FailEnd;
}
else
return ICResult.FailUpdate;
}
}
Regarding lpType - in C++, you pass RT_ICON (or RT_GROUP_ICON). What value should I pass in C#?
The same question goes for lpName parameter.
I'm not sure about language identifier (I found this on Internet) since I cannot test it.
I'm also not sure whether I'm providing appropriate icon data. Currently, iconData contains the bytes from .ico file.
Can anybody point me to right direction?
Thank you very much.

Just some pointers, this is quite hard to get right. Pass an RT_ICON by lying about the lpType argument. Change it from string to IntPtr and pass (IntPtr)3.
The lpData argument is quite tricky. You need to pass the data the way it is compiled by the resource compiler (rc.exe). I have no idea if it mangles the raw data of the .ico file. The only reasonable thing to try is to read the data from the .ico file with FileStream into a byte[], you already seem to be doing this. I think the function was really designed to copy a resource from one binary image to another. Odds of your approach working are not zero.
You are also ignoring another potential problem, the resource ID of the icon of the program isn't necessarily 1. It often is not, 100 tends to be a popular choice, but anything goes. EnumResourceNames would be required to make it reliable. The rule is that the lowest numbered ID sets the icon for the file. I'm not actually sure if that really means that the resource compiler puts the lowest number first, something that the API probably doesn't do.
A very small failure mode is that UpdateResource can only updated numbered resource items, not named ones. Using names instead of numbers is not uncommon but the vast majority of images use numbers for icons.
And of course, the odds that this will work without a UAC manifest are zero. You are hacking files that you don't normally have write access to.

I managed to get this working in pure C# using ResourceHacker and this posting as an example. Just use a regular .ico as input. In ResourceHacker (http://www.angusj.com/resourcehacker/) you will see the icon identifier (in my case 1) and the language identifier (in my case 1043):
I used this code:
internal class IconChanger
{
#region IconReader
public class Icons : List<Icon>
{
public byte[] ToGroupData(int startindex = 1)
{
using (var ms = new MemoryStream())
using (var writer = new BinaryWriter(ms))
{
var i = 0;
writer.Write((ushort)0); //reserved, must be 0
writer.Write((ushort)1); // type is 1 for icons
writer.Write((ushort)this.Count); // number of icons in structure(1)
foreach (var icon in this)
{
writer.Write(icon.Width);
writer.Write(icon.Height);
writer.Write(icon.Colors);
writer.Write((byte)0); // reserved, must be 0
writer.Write(icon.ColorPlanes);
writer.Write(icon.BitsPerPixel);
writer.Write(icon.Size);
writer.Write((ushort)(startindex + i));
i++;
}
ms.Position = 0;
return ms.ToArray();
}
}
}
public class Icon
{
public byte Width { get; set; }
public byte Height { get; set; }
public byte Colors { get; set; }
public uint Size { get; set; }
public uint Offset { get; set; }
public ushort ColorPlanes { get; set; }
public ushort BitsPerPixel { get; set; }
public byte[] Data { get; set; }
}
public class IconReader
{
public Icons Icons = new Icons();
public IconReader(Stream input)
{
using (BinaryReader reader = new BinaryReader(input))
{
reader.ReadUInt16(); // ignore. Should be 0
var type = reader.ReadUInt16();
if (type != 1)
{
throw new Exception("Invalid type. The stream is not an icon file");
}
var num_of_images = reader.ReadUInt16();
for (var i = 0; i < num_of_images; i++)
{
var width = reader.ReadByte();
var height = reader.ReadByte();
var colors = reader.ReadByte();
reader.ReadByte(); // ignore. Should be 0
var color_planes = reader.ReadUInt16(); // should be 0 or 1
var bits_per_pixel = reader.ReadUInt16();
var size = reader.ReadUInt32();
var offset = reader.ReadUInt32();
this.Icons.Add(new Icon()
{
Colors = colors,
Height = height,
Width = width,
Offset = offset,
Size = size,
ColorPlanes = color_planes,
BitsPerPixel = bits_per_pixel
});
}
// now get the Data
foreach (var icon in Icons)
{
if (reader.BaseStream.Position < icon.Offset)
{
var dummy_bytes_to_read = (int)(icon.Offset - reader.BaseStream.Position);
reader.ReadBytes(dummy_bytes_to_read);
}
var data = reader.ReadBytes((int)icon.Size);
icon.Data = data;
}
}
}
}
#endregion
[DllImport("kernel32.dll", SetLastError = true)]
static extern int UpdateResource(IntPtr hUpdate, uint lpType, ushort lpName, ushort wLanguage, byte[] lpData, uint cbData);
[DllImport("kernel32.dll", SetLastError = true)]
static extern IntPtr BeginUpdateResource(string pFileName, [MarshalAs(UnmanagedType.Bool)]bool bDeleteExistingResources);
[DllImport("kernel32.dll", SetLastError = true)]
static extern bool EndUpdateResource(IntPtr hUpdate, bool fDiscard);
public enum ICResult
{
Success,
FailBegin,
FailUpdate,
FailEnd
}
const uint RT_ICON = 3;
const uint RT_GROUP_ICON = 14;
public ICResult ChangeIcon(string exeFilePath, string iconFilePath)
{
using (FileStream fs = new FileStream(iconFilePath, FileMode.Open, FileAccess.Read))
{
var reader = new IconReader(fs);
var iconChanger = new IconChanger();
return iconChanger.ChangeIcon(exeFilePath, reader.Icons);
}
}
public ICResult ChangeIcon(string exeFilePath, Icons icons)
{
// Load executable
IntPtr handleExe = BeginUpdateResource(exeFilePath, false);
if (handleExe == null) return ICResult.FailBegin;
ushort startindex = 1;
ushort index = startindex;
ICResult result = ICResult.Success;
var ret = 1;
foreach (var icon in icons)
{
// Replace the icon
// todo :Improve the return value handling of UpdateResource
ret = UpdateResource(handleExe, RT_ICON, index, 0, icon.Data, icon.Size);
index++;
}
var groupdata = icons.ToGroupData();
// todo :Improve the return value handling of UpdateResource
ret = UpdateResource(handleExe, RT_GROUP_ICON, startindex, 0, groupdata, (uint)groupdata.Length);
if (ret == 1)
{
if (EndUpdateResource(handleExe, false))
result = ICResult.Success;
else
result = ICResult.FailEnd;
}
else
result = ICResult.FailUpdate;
return result;
}
}

This is the solution that worked for me. I was not able to write it in .NET, but have managed to write C++ DLL which I am referencing in my C# application.
C++ DLL
The contents of C++ solution I am building the DLL from:
#include <io.h>
#include <stdio.h>
#include <stdlib.h>
#include <fcntl.h>
#include <windows.h>
extern "C"
{
#pragma pack(push, 2)
typedef struct {
WORD Reserved1; // reserved, must be 0
WORD ResourceType; // type is 1 for icons
WORD ImageCount; // number of icons in structure (1)
BYTE Width; // icon width (32)
BYTE Height; // icon height (32)
BYTE Colors; // colors (0 means more than 8 bits per pixel)
BYTE Reserved2; // reserved, must be 0
WORD Planes; // color planes
WORD BitsPerPixel; // bit depth
DWORD ImageSize; // size of structure
WORD ResourceID; // resource ID
} GROUPICON;
#pragma pack(pop)
__declspec(dllexport) void __stdcall ChangeIcon(char *executableFile, char *iconFile, INT16 imageCount)
{
int len = strlen(executableFile) + 1;
wchar_t *executableFileEx = new wchar_t[len];
memset(executableFileEx, 0, len);
::MultiByteToWideChar(CP_ACP, NULL, executableFile, -1, executableFileEx, len);
len = strlen("MAINICON") + 1;
wchar_t *mainIconEx = new wchar_t[len];
memset(mainIconEx, 0, len);
::MultiByteToWideChar(CP_ACP, NULL, "MAINICON", -1, mainIconEx, len);
HANDLE hWhere = BeginUpdateResource(executableFileEx, FALSE);
char *buffer; // Buffer to store raw icon data
long buffersize; // Length of buffer
int hFile; // File handle
hFile = open(iconFile, O_RDONLY | O_BINARY);
if (hFile == -1)
return; // If file doesn't exist, can't be opened etc.
// Calculate buffer length and load file into buffer
buffersize = filelength(hFile);
buffer = (char *)malloc(buffersize);
read(hFile, buffer, buffersize);
close(hFile);
// Calculate header size
int headerSize = 6 + imageCount * 16;
UpdateResource(
hWhere, // Handle to executable
RT_ICON, // Resource type - icon
MAKEINTRESOURCE(1), // Make the id 1
MAKELANGID(LANG_ENGLISH, SUBLANG_DEFAULT), // Default language
buffer + headerSize, // Skip the header bytes
buffersize - headerSize // Length of buffer
);
GROUPICON grData;
grData.Reserved1 = 0; // reserved, must be 0
grData.ResourceType = 1; // type is 1 for icons
grData.ImageCount = 1; // number of icons in structure (1)
grData.Width = 32; // icon width (32)
grData.Height = 32; // icon height (32)
grData.Colors = 0; // colors (256)
grData.Reserved2 = 0; // reserved, must be 0
grData.Planes = 2; // color planes
grData.BitsPerPixel = 32; // bit depth
grData.ImageSize = buffersize - 22; // size of image
grData.ResourceID = 1; // resource ID is 1
UpdateResource(
hWhere,
RT_GROUP_ICON,
// RT_GROUP_ICON resources contain information
// about stored icons
mainIconEx,
// MAINICON contains information about the
// application's displayed icon
MAKELANGID(LANG_ENGLISH, SUBLANG_DEFAULT),
&grData,
// Pointer to this structure
sizeof(GROUPICON)
);
delete buffer; // free memory
// Perform the update, don't discard changes
EndUpdateResource(hWhere, FALSE);
}
}
C# code
This is C# code which I'm using to import ChangeIcon function from previously written DLL:
[DllImport("IconChanger.dll")]
static extern void ChangeIcon(String executableFile, String iconFile, short imageCount);
/// <summary>
/// Changes the executable's icon
/// </summary>
/// <param name="exeFilePath">Path to executable file</param>
/// <param name="iconFilePath">Path to icon file</param>
static public void ChangeIcon(string exeFilePath, string iconFilePath)
{
short imageCount = 0;
using (StreamReader sReader = new StreamReader(iconFilePath))
{
using (BinaryReader bReader = new BinaryReader(sReader.BaseStream))
{
// Retrieve icon count inside icon file
bReader.ReadInt16();
bReader.ReadInt16();
imageCount = bReader.ReadInt16();
}
}
// Change the executable's icon
ChangeIcon(exeFilePath, iconFilePath, imageCount);
}
Hope at least somebody will find this useful.

C# declaration for UpdateResource:
/// <summary>
/// Adds, deletes, or replaces a resource in a portable executable (PE) file. There are some restrictions on resource updates in files that contain Resource Configuration (RC Config) data: language-neutral (LN) files and language-specific resource (.mui) files.
/// </summary>
/// <param name="hUpdate">A module handle returned by the BeginUpdateResource function, referencing the file to be updated. </param>
/// <param name="lpType">The resource type to be updated. Alternatively, rather than a pointer, this parameter can be MAKEINTRESOURCE(ID), where ID is an integer value representing a predefined resource type. If the first character of the string is a pound sign (#), then the remaining characters represent a decimal number that specifies the integer identifier of the resource type. For example, the string "#258" represents the identifier 258. For a list of predefined resource types, see Resource Types. </param>
/// <param name="lpName">The name of the resource to be updated. Alternatively, rather than a pointer, this parameter can be MAKEINTRESOURCE(ID), where ID is a resource ID. When creating a new resource do not use a string that begins with a '#' character for this parameter.</param>
/// <param name="wLanguage">The language identifier of the resource to be updated. For a list of the primary language identifiers and sublanguage identifiers that make up a language identifier, see the MAKELANGID macro. </param>
/// <param name="lpData">The resource data to be inserted into the file indicated by hUpdate. If the resource is one of the predefined types, the data must be valid and properly aligned. Note that this is the raw binary data to be stored in the file indicated by hUpdate, not the data provided by LoadIcon, LoadString, or other resource-specific load functions. All data containing strings or text must be in Unicode format. lpData must not point to ANSI data. If lpData is NULL and cbData is 0, the specified resource is deleted from the file indicated by hUpdate. Prior to Windows 7: If lpData is NULL and cbData is nonzero, the specified resource is NOT deleted and an exception is thrown.</param>
/// <param name="cbData">The size, in bytes, of the resource data at lpData. </param>
/// <returns>Returns TRUE if successful or FALSE otherwise. To get extended error information, call GetLastError.</returns>
[DllImport("kernel32.dll", CharSet = CharSet.Unicode, PreserveSig = true, SetLastError = true, ExactSpelling = true)]
public static extern Int32 UpdateResourceW(void* hUpdate, char* lpType, char* lpName, UInt16 wLanguage, [CanBeNull] void* lpData, UInt32 cbData);
For a string resource type or name, you just pass the string.
For system-predefined types like RT_ICON and int IDs like IDI_APPLICATION, you pass that integer value reinterpret-casting it to a pointer, like (char*)3 for RT_ICON.