We Keep Coding

How to read Memo type field from DBF file in C#?

I am unable to find the solution that how to read Memo type field from DBF file.
As per my current RnD, I found out that the "Memo" data is stored in a separate file with the same name as the DBF file, but with a ".fpt" file extension. The "Memo" data file consists of a series of blocks, each of which contains a fixed number of bytes (typically 512 bytes), and each block is identified by a block number.
The "Memo" column in the DBF file contains a 10-byte pointer that identifies the block number of the first block of data for the corresponding record. If the data for a particular record exceeds the size of one block, additional blocks are allocated as needed and linked together using the pointers stored in the first block.
Moreover I used the source code provided here
using System;
using System.Collections.Generic;
using System.Data;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Runtime.InteropServices;
using System.Text;
namespace System.IO
{
/// <summary>
/// This class reads a dbf files
/// </summary>
public class DBFReader : IDisposable
{
private BinaryReader reader;
private Encoding encoding;
public DBFReader(Stream stream, Encoding encoding)
{
this.encoding = encoding;
this.reader = new BinaryReader(stream, encoding);
ReadHeader();
}
public DBFReader(string filename, Encoding encoding)
{
if (File.Exists(filename) == false)
throw new FileNotFoundException();
this.encoding = encoding;
var bs = new BufferedStream(File.OpenRead(filename));
this.reader = new BinaryReader(bs, encoding);
ReadHeader();
}
private void ReadHeader()
{
byte[] buffer = reader.ReadBytes(Marshal.SizeOf(typeof(DBFHeader)));
// Marshall the header into a DBFHeader structure
GCHandle handle = GCHandle.Alloc(buffer, GCHandleType.Pinned);
this.header = (DBFHeader)Marshal.PtrToStructure(handle.AddrOfPinnedObject(), typeof(DBFHeader));
handle.Free();
fields = new List<DBFFieldDescriptor>();
while (reader.PeekChar() != 13)
{
buffer = reader.ReadBytes(Marshal.SizeOf(typeof(DBFFieldDescriptor)));
handle = GCHandle.Alloc(buffer, GCHandleType.Pinned);
var fieldDescriptor = (DBFFieldDescriptor)Marshal.PtrToStructure(handle.AddrOfPinnedObject(), typeof(DBFFieldDescriptor));
if ((fieldDescriptor.Flags & DBFFieldFlags.System) != DBFFieldFlags.System )
{
fields.Add(fieldDescriptor);
}
handle.Free();
}
byte headerTerminator = reader.ReadByte();
byte[] backlink = reader.ReadBytes(263);
}
private void ReadRecords()
{
records = new List<Dictionary<DBFFieldDescriptor, object>>();
// Skip back to the end of the header.
reader.BaseStream.Seek(header.HeaderLenght, SeekOrigin.Begin);
for (int i = 0; i < header.NumberOfRecords; i++)
{
if (reader.PeekChar() == '*') // DELETED
{
continue;
}
var record = new Dictionary<DBFFieldDescriptor, object>();
var row = reader.ReadBytes(header.RecordLenght);
foreach (var field in fields)
{
byte[] buffer = new byte[field.FieldLength];
Array.Copy(row, field.Address, buffer, 0, field.FieldLength);
string text = (encoding.GetString(buffer) ?? String.Empty).Trim();
switch ((DBFFieldType)field.FieldType)
{
case DBFFieldType.Character:
record[field] = text;
break;
case DBFFieldType.Currency:
if (String.IsNullOrWhiteSpace(text))
{
if ((field.Flags & DBFFieldFlags.AllowNullValues) == DBFFieldFlags.AllowNullValues)
{
record[field] = null;
}
else
{
record[field] = 0.0m;
}
}
else
{
record[field] = Convert.ToDecimal(text);
}
break;
case DBFFieldType.Numeric:
case DBFFieldType.Float:
if (String.IsNullOrWhiteSpace(text))
{
if ((field.Flags & DBFFieldFlags.AllowNullValues) == DBFFieldFlags.AllowNullValues)
{
record[field] = null;
}
else
{
record[field] = 0.0f;
}
}
else
{
record[field] = Convert.ToSingle(text);
}
break;
case DBFFieldType.Date:
if (String.IsNullOrWhiteSpace(text))
{
if ((field.Flags & DBFFieldFlags.AllowNullValues) == DBFFieldFlags.AllowNullValues)
{
record[field] = null;
}
else
{
record[field] = DateTime.MinValue;
}
}
else
{
record[field] = DateTime.ParseExact(text, "yyyyMMdd", CultureInfo.InvariantCulture);
}
break;
case DBFFieldType.DateTime:
if (String.IsNullOrWhiteSpace(text) || BitConverter.ToInt64(buffer, 0) == 0)
{
if ((field.Flags & DBFFieldFlags.AllowNullValues) == DBFFieldFlags.AllowNullValues)
{
record[field] = null;
}
else
{
record[field] = DateTime.MinValue;
}
}
else
{
record[field] = JulianToDateTime(BitConverter.ToInt64(buffer, 0));
}
break;
case DBFFieldType.Double:
if (String.IsNullOrWhiteSpace(text))
{
if ((field.Flags & DBFFieldFlags.AllowNullValues) == DBFFieldFlags.AllowNullValues)
{
record[field] = null;
}
else
{
record[field] = 0.0;
}
}
else
{
record[field] = Convert.ToDouble(text);
}
break;
case DBFFieldType.Integer:
if (String.IsNullOrWhiteSpace(text))
{
if ((field.Flags & DBFFieldFlags.AllowNullValues) == DBFFieldFlags.AllowNullValues)
{
record[field] = null;
}
else
{
record[field] = 0;
}
}
else
{
record[field] = BitConverter.ToInt32(buffer, 0);
}
break;
case DBFFieldType.Logical:
if (String.IsNullOrWhiteSpace(text))
{
if ((field.Flags & DBFFieldFlags.AllowNullValues) == DBFFieldFlags.AllowNullValues)
{
record[field] = null;
}
else
{
record[field] = false;
}
}
else
{
record[field] = (buffer[0] == 'Y' || buffer[0] == 'T');
}
break;
case DBFFieldType.Memo:
case DBFFieldType.General:
case DBFFieldType.Picture:
default:
record[field] = buffer;
break;
}
}
records.Add(record);
}
}
public DataTable ReadToDataTable()
{
ReadRecords();
var table = new DataTable();
// Columns
foreach (var field in fields)
{
var colType = ToDbType(field.FieldType);
var column = new DataColumn(field.FieldName, colType ?? typeof(String));
table.Columns.Add(column);
}
// Rows
foreach (var record in records)
{
var row = table.NewRow();
foreach (var column in record.Keys)
{
row[column.FieldName] = record[column] ?? DBNull.Value;
}
table.Rows.Add(row);
}
return table;
}
public IEnumerable<Dictionary<string, object>> ReadToDictionary()
{
ReadRecords();
return records.Select(record => record.ToDictionary(r => r.Key.FieldName, r => r.Value)).ToList();
}
public IEnumerable<T> ReadToObject<T>()
where T : new()
{
ReadRecords();
var type = typeof(T);
var list = new List<T>();
foreach (var record in records)
{
T item = new T();
foreach (var pair in record.Select(s => new { Key = s.Key.FieldName, Value = s.Value }))
{
var property = type.GetProperty(pair.Key, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance);
if (property != null)
{
if (property.PropertyType == pair.Value.GetType())
{
property.SetValue(item, pair.Value, null);
}
else
{
if (pair.Value != DBNull.Value)
{
property.SetValue(item, System.Convert.ChangeType(pair.Value, property.PropertyType), null);
}
}
}
}
list.Add(item);
}
return list;
}
private DBFHeader header;
private List<DBFFieldDescriptor> fields = new List<DBFFieldDescriptor>();
private List<Dictionary<DBFFieldDescriptor, object>> records = new List<Dictionary<DBFFieldDescriptor,object>>();
#region IDisposable
public void Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}
protected void Dispose(bool disposing)
{
if (disposing == false) return;
if (reader != null)
{
reader.Close();
reader.Dispose();
reader = null;
}
}
~DBFReader()
{
Dispose(false);
}
#endregion
/// <summary>
/// Convert a Julian Date as long to a .NET DateTime structure
/// Implemented from pseudo code at http://en.wikipedia.org/wiki/Julian_day
/// </summary>
/// <param name="julianDateAsLong">Julian Date to convert (days since 01/01/4713 BC)</param>
/// <returns>DateTime</returns>
private static DateTime JulianToDateTime(long julianDateAsLong)
{
if (julianDateAsLong == 0) return DateTime.MinValue;
double p = Convert.ToDouble(julianDateAsLong);
double s1 = p + 68569;
double n = Math.Floor(4 * s1 / 146097);
double s2 = s1 - Math.Floor(((146097 * n) + 3) / 4);
double i = Math.Floor(4000 * (s2 + 1) / 1461001);
double s3 = s2 - Math.Floor(1461 * i / 4) + 31;
double q = Math.Floor(80 * s3 / 2447);
double d = s3 - Math.Floor(2447 * q / 80);
double s4 = Math.Floor(q / 11);
double m = q + 2 - (12 * s4);
double j = (100 * (n - 49)) + i + s4;
return new DateTime(Convert.ToInt32(j), Convert.ToInt32(m), Convert.ToInt32(d));
}
/// <summary>
/// This is the file header for a DBF. We do this special layout with everything
/// packed so we can read straight from disk into the structure to populate it
/// </summary>
[StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi, Pack = 1)]
private struct DBFHeader
{
/// <summary>The version.</summary>
public readonly DBFVersion Version;
/// <summary>The update year.</summary>
public readonly byte UpdateYear;
/// <summary>The update month.</summary>
public readonly byte UpdateMonth;
/// <summary>The update day.</summary>
public readonly byte UpdateDay;
/// <summary>The number of records.</summary>
public readonly int NumberOfRecords;
/// <summary>The length of the header.</summary>
public readonly short HeaderLenght;
/// <summary>The length of the bytes records.</summary>
public readonly short RecordLenght;
[MarshalAs(UnmanagedType.ByValArray, SizeConst = 16)]
public readonly byte[] Reserved;
/// <summary>Table Flags</summary>
public readonly DBFTableFlags TableFlags;
/// <summary>Code Page Mark</summary>
public readonly byte CodePage;
/// <summary>Reserved, contains 0x00</summary>
public readonly short EndOfHeader;
}
public enum DBFVersion : byte
{
Unknown = 0,
FoxBase = 0x02,
FoxBaseDBase3NoMemo = 0x03,
VisualFoxPro = 0x30,
VisualFoxProWithAutoIncrement = 0x31,
dBase4SQLTableNoMemo = 0x43,
dBase4SQLSystemNoMemo = 0x63,
FoxBaseDBase3WithMemo = 0x83,
dBase4WithMemo = 0x8B,
dBase4SQLTableWithMemo = 0xCB,
FoxPro2WithMemo = 0xF5,
FoxBASE = 0xFB
}
[Flags]
public enum DBFTableFlags : byte
{
None = 0x00,
HasStructuralCDX = 0x01,
HasMemoField = 0x02,
IsDBC = 0x04
}
/// <summary>
/// This is the field descriptor structure. There will be one of these for each column in the table.
/// </summary>
[StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi, Pack = 1)]
private struct DBFFieldDescriptor
{
/// <summary>The field name.</summary>
[MarshalAs(UnmanagedType.ByValTStr, SizeConst = 11)]
public readonly string FieldName;
/// <summary>The field type.</summary>
public readonly char FieldType;
/// <summary>The field address.</summary>
public readonly int Address;
/// <summary>The field length in bytes.</summary>
public readonly byte FieldLength;
/// <summary>The field precision.</summary>
public readonly byte DecimalCount;
/// <summary>Field Flags</summary>
public readonly DBFFieldFlags Flags;
/// <summary>AutoIncrement next value</summary>
public readonly int AutoIncrementNextValue;
/// <summary>AutoIncrement step value</summary>
public readonly byte AutoIncrementStepValue;
/// <summary>Reserved</summary>
[MarshalAs(UnmanagedType.ByValArray, SizeConst = 8)]
public readonly byte[] Reserved;
public override string ToString()
{
return String.Format("{0} {1}", FieldName, FieldType);
}
}
[Flags]
public enum DBFFieldFlags : byte
{
None = 0x00,
System = 0x01,
AllowNullValues = 0x02,
Binary = 0x04,
AutoIncrementing = 0x0C
}
public enum DBFFieldType : int
{
Character = 'C',
Currency = 'Y',
Numeric = 'N',
Float = 'F',
Date = 'D',
DateTime = 'T',
Double = 'B',
Integer = 'I',
Logical = 'L',
Memo = 'M',
General = 'G',
Picture = 'P'
}
public static Type ToDbType(char type)
{
switch ((DBFFieldType)type)
{
case DBFFieldType.Float:
return typeof(float);
case DBFFieldType.Integer:
return typeof(int);
case DBFFieldType.Currency:
return typeof(decimal);
case DBFFieldType.Character:
case DBFFieldType.Memo:
return typeof(string);
case DBFFieldType.Date:
case DBFFieldType.DateTime:
return typeof(DateTime);
case DBFFieldType.Logical:
return typeof(bool);
case DBFFieldType.General:
case DBFFieldType.Picture:
return typeof(byte[]);
default:
return null;
}
}
}
}

What is your question?
(Would be a mess as comment)
Your information about FPT file structure is not entirely correct and full details exist in the VFP help file.
Do you have to read it low level? If you have to, basically what you do is:
Collect memo block numbers from the DBF itself (block numbers are stored in corresponding field in DBF),
For a memo entry, go to block number in FPT (depends on block size)
From the block header read the size and get the bytes that matches the stored size.
While doing that, always read as binary, because unlike C strings, VFP string values are not ASCIIZ values, they can even contain '\x0'.
If it is not a must to read that way then the easy way is to simply read it using VFPOLEDB OleDb driver. ie:
void Main()
{
string cn = #"Provider=VFPOLEDB;Data source=C:\PROGRAM FILES (X86)\MICROSOFT VISUAL FOXPRO 9\SAMPLES\DATA;";
string query = "select emp_id, first_name, last_name, notes from Employee";
DataTable t = new DataTable();
new OleDbDataAdapter(query, cn).Fill(t);
foreach (var row in t.AsEnumerable())
{
Console.WriteLine($"{row["emp_id"]}, {row["first_name"]}, {row["last_name"]}, ({row["notes"]})");
}
}
Note that VFPOLEDB driver is 32 bits, you would need to target x86 platform. There are 64 bits drivers from Sybase ADS as they say (have never used).
PS: Also search for Tom Brother's Linq To VFp, Linq To EF VFP drivers.
PS2: I quickly glanced the source code you provided, and beware that is for older VFP versions and is not really correct (ie: Assumes Memo is a string which is not correct - if you accept them string in C# you are likely going to lose many data).

Get friendly name of a device from WM_DEVICECHANGE LParam

So i'm building an app for a school PC that tracks all inserted devices.
I managed to use RegisterDeviceNotification() to get a notification in the main thread whenever i plug in or remove a device.
All i can get though, is the LParam, a pointer that is device unique.
I can't find anything about how to get the friendly name of the device using that LParam.
The only resource i can find is this CodeProject from 2006 (in C++).
I can't find anything on pinvoke.net and the only thing i found (i don't remember exactly where) is using a ManagementObjectSearcher to get this data, but it finds null data. (here's the code)
ManagementObjectSearcher searcher = new ManagementObjectSearcher("Select Name from Win32_PnpEntity");
foreach (ManagementObject devices in searcher.Get())
{
foreach (var v in devices.Properties)
{
Console.WriteLine(v.Value);
}
}
searcher.Dispose();
Can anyone please help me figure out how to get the friendly name of the device?

you need register device notifications with
RegisterDeviceNotificationW
static DEV_BROADCAST_DEVICEINTERFACE dbd = { sizeof(dbd), DBT_DEVTYP_DEVICEINTERFACE };
_hDevNot = RegisterDeviceNotificationW(hwnd, &dbd,
DEVICE_NOTIFY_WINDOW_HANDLE|DEVICE_NOTIFY_ALL_INTERFACE_CLASSES);
as result you will receive WM_DEVICECHANGE message
here you need check wParam for for DBT_DEVICEARRIVAL and
DBT_DEVICEREMOVECOMPLETE events
check that dbcc_devicetype == DBT_DEVTYP_DEVICEINTERFACE
inside DEV_BROADCAST_DEVICEINTERFACE_W.dbcc_name you got
device interface name
you can use this name inside CM_Get_Device_Interface_PropertyW
function with DEVPKEY_Device_InstanceId on
DBT_DEVICEARRIVAL event
you got Device Instance ID
use it in call CM_Locate_DevNodeW
use result in call CM_Get_DevNode_PropertyW with
DEVPKEY_NAME or DEVPKEY_DeviceInterface_FriendlyName
on DBT_DEVICEREMOVECOMPLETE - you can only look in self
database (if you create it on arrival) for such name and friendly
name.
demo code:
struct DeviceName : public LIST_ENTRY
{
ULONG InterfaceHash;
WCHAR Name[];
void* operator new(size_t cb, size_t len)
{
return LocalAlloc(0, cb + len);
}
void operator delete(void* pv)
{
LocalFree(pv);
}
};
volatile const UCHAR guz = 0;
CONFIGRET GetFriendlyNameByDevNode(DeviceName** pp, DEVINST dnDevInst)
{
CONFIGRET status;
ULONG cb = 32;
DEVPROPTYPE PropertyType;
do
{
if (DeviceName* p = new(cb) DeviceName)
{
status = CM_Get_DevNode_PropertyW(
dnDevInst, &DEVPKEY_DeviceInterface_FriendlyName, &PropertyType, (PBYTE)p->Name, &cb, 0);
if (status == CR_SUCCESS)
{
if (PropertyType == DEVPROP_TYPE_STRING)
{
*pp = p;
return CR_SUCCESS;
}
else
{
status = CR_WRONG_TYPE;
}
}
delete p;
}
else
{
status = CR_OUT_OF_MEMORY;
}
} while (CR_BUFFER_SMALL == status);
return status;
}
CONFIGRET GetFriendlyNameByInterface(DeviceName** pp, PCWSTR pszDeviceInterface)
{
// RTCu must be disabled !
ULONG cb = 0, rcb = 64;
PVOID stack = alloca(guz);
DEVPROPTYPE PropertyType;
CONFIGRET status;
union {
PVOID pv;
PWSTR DeviceID;
PBYTE pb;
};
do
{
if (cb < rcb)
{
rcb = cb = RtlPointerToOffset(pv = alloca(rcb - cb), stack);
}
} while (CR_BUFFER_SMALL == (status = CM_Get_Device_Interface_PropertyW(
pszDeviceInterface, &DEVPKEY_Device_InstanceId, &PropertyType, pb, &rcb, 0)));
if (status == CR_SUCCESS)
{
if (PropertyType == DEVPROP_TYPE_STRING)
{
DEVINST dnDevInst;
status = CM_Locate_DevNodeW(&dnDevInst, DeviceID, CM_LOCATE_DEVNODE_NORMAL);
return status == CR_SUCCESS ? GetFriendlyNameByDevNode(pp, dnDevInst) : status;
}
else
{
status = CR_WRONG_TYPE;
}
}
return status;
}
case WM_DESTROY:
if (_hDevNot)
{
UnregisterDeviceNotification(_hDevNot);
PLIST_ENTRY head = &_DevListHead, entry = head->Flink;
while (entry != head)
{
DeviceName* p = static_cast<DeviceName*>(entry);
entry = entry->Flink;
delete p;
}
}
break;
case WM_DEVICECHANGE:
switch (wParam)
{
case DBT_DEVICEREMOVECOMPLETE:
case DBT_DEVICEARRIVAL:
if (reinterpret_cast<PDEV_BROADCAST_DEVICEINTERFACE>(lParam)->dbcc_devicetype == DBT_DEVTYP_DEVICEINTERFACE)
{
DeviceName* p;
ULONG InterfaceHash;
UNICODE_STRING dbcc_name;
RtlInitUnicodeString(&dbcc_name, reinterpret_cast<PDEV_BROADCAST_DEVICEINTERFACE>(lParam)->dbcc_name);
RtlHashUnicodeString(&dbcc_name, FALSE, HASH_STRING_ALGORITHM_DEFAULT, &InterfaceHash);
if (wParam == DBT_DEVICEARRIVAL)
{
if (CR_SUCCESS == GetFriendlyNameByInterface(&p, dbcc_name.Buffer))
{
p->InterfaceHash = InterfaceHash;
InsertHeadList(&_DevListHead, p);
DbgPrint("inserted %S ( %wZ )\n", p->Name, &dbcc_name);
}
}
else
{
PLIST_ENTRY head = &_DevListHead, entry = head;
while ((entry = entry->Flink) != head)
{
if (static_cast<DeviceName*>(entry)->InterfaceHash == InterfaceHash)
{
DbgPrint("removed %S ( %wZ )\n",
static_cast<DeviceName*>(entry)->Name, &dbcc_name);
RemoveEntryList(entry);
delete static_cast<DeviceName*>(entry);
break;
}
}
}
}
break;
}
return 0;
case WM_CREATE:
InitializeListHead(&_DevListHead);
static DEV_BROADCAST_DEVICEINTERFACE dbd = { sizeof(dbd), DBT_DEVTYP_DEVICEINTERFACE };
_hDevNot = RegisterDeviceNotificationW(hwnd, &dbd, DEVICE_NOTIFY_WINDOW_HANDLE|DEVICE_NOTIFY_ALL_INTERFACE_CLASSES);
break;

Rx.Net Message Parser

I'm trying to parse an incoming stream of bytes that represents messages.
I need to split the stream and create a message structure for each part.
A message always starts with a 0x81 (BOM) and ends with a 0x82 (EOM).
start: 0x81
header: 3 bytes
data: arbitrary length
stop: 0x82
The data part is escaped using an escape byte 0x1B (ESC): Anytime a byte in the data part contains one of the control bytes {ESC, BOM, EOM}, it is prefixed with ESC.
The header part is not escaped, and may contain control bytes.
I would like to code this in a functional reactive style using Rx.Net, by consuming an IObservable<byte> and transforming it into an IObservable<Message>.
What is the most idiomatic way to do this?
Some examples:
[81 01 02 03 82] single message
[81 82 81 82 82] single message, header = [82 81 82]
[81 01 02 1B 82] single message, header = [01 02 1B].
[81 01 02 03 1B 82 82] single message, header = [01 02 03], (unescaped) data = [82]
[81 01 02 03 1B 1B 82 82] single message + dangling [82] which should be ignored.
header = [01 02 03], (unescaped) data = [1B]
Here's a state machine drawing for this:

If you are looking for something the is "more functional" then this may help, however the answer by #Evk pass these tests too.
Firstly can I suggest, to take the leg work out of providing a verifiable answer, could you provide a test suite to implement on complex questions like this.
Something like this would have been very helpful.
var scheduler = new TestScheduler();
var source = scheduler.CreateColdObservable<byte>(
ReactiveTest.OnNext<byte>(01,0x81), //BOM m1
ReactiveTest.OnNext<byte>(02,0x01),
ReactiveTest.OnNext<byte>(03,0x02),
ReactiveTest.OnNext<byte>(04,0x03),
ReactiveTest.OnNext<byte>(05,0x82), //EOM m1
ReactiveTest.OnNext<byte>(06,0x81), //BOM m2
ReactiveTest.OnNext<byte>(07,0x82),
ReactiveTest.OnNext<byte>(08,0x81),
ReactiveTest.OnNext<byte>(09,0x82),
ReactiveTest.OnNext<byte>(10,0x82), //EOM m2
ReactiveTest.OnNext<byte>(11,0x81), //BOM m3
ReactiveTest.OnNext<byte>(12,0x01),
ReactiveTest.OnNext<byte>(13,0x02),
ReactiveTest.OnNext<byte>(14,0x1B),
ReactiveTest.OnNext<byte>(15,0x82), //EOM m3
ReactiveTest.OnNext<byte>(16,0x81), //BOM m4
ReactiveTest.OnNext<byte>(17,0x01),
ReactiveTest.OnNext<byte>(18,0x02),
ReactiveTest.OnNext<byte>(19,0x03),
ReactiveTest.OnNext<byte>(20,0x1B), //Control character
ReactiveTest.OnNext<byte>(21,0x82), //Data
ReactiveTest.OnNext<byte>(22,0x82), //EOM m4
ReactiveTest.OnNext<byte>(23,0x81), //BOM m5
ReactiveTest.OnNext<byte>(24,0x01),
ReactiveTest.OnNext<byte>(25,0x02),
ReactiveTest.OnNext<byte>(26,0x03),
ReactiveTest.OnNext<byte>(27,0x1B), //Control character
ReactiveTest.OnNext<byte>(28,0x1B), //Data
ReactiveTest.OnNext<byte>(29,0x82), //EOM m5
ReactiveTest.OnNext<byte>(30,0x82));//Ignored (expected 0x81)
var observer = scheduler.CreateObserver<Message>();
//CurrentAnswer(source)
MyAnswer(source)
.Subscribe(observer);
scheduler.Start();
ReactiveAssert.AreElementsEqual(
new[] {
ReactiveTest.OnNext(05, new Message(){Header=new byte[]{0x01, 0x02, 0x03}, Data=new byte[0]{}}),
ReactiveTest.OnNext(10, new Message(){Header=new byte[]{0x82, 0x81, 0x82}, Data=new byte[0]{}}),
ReactiveTest.OnNext(15, new Message(){Header=new byte[]{0x01, 0x02, 0x1B}, Data=new byte[0]{}}),
ReactiveTest.OnNext(22, new Message(){Header=new byte[]{0x01, 0x02, 0x03}, Data=new byte[]{ 0x82}}),
ReactiveTest.OnNext(29, new Message(){Header=new byte[]{0x01, 0x02, 0x03}, Data=new byte[]{ 0x1B}}),
},
observer.Messages);
I have also written a version of Message that allows me to verify the code
public class Message
{
public static readonly byte BOM = 0x81;
public static readonly byte EOM = 0x82;
public static readonly byte Control = 0x1B;
public byte[] Header { get; set; }
public byte[] Data { get; set; }
public static Message Create(byte[] bytes)
{
if(bytes==null)
throw new ArgumentNullException(nameof(bytes));
if(bytes.Length<3)
throw new ArgumentException("bytes<3").Dump();
var header = new byte[3];
Array.Copy(bytes, header, 3);
var body = new List<byte>();
var escapeNext = false;
for (int i = 3; i < bytes.Length; i++)
{
var b = bytes[i];
if (b == Control && !escapeNext)
{
escapeNext = true;
}
else
{
body.Add(b);
escapeNext = false;
}
}
var msg = new Message { Header = header, Data = body.ToArray()};
return msg;
}
public override string ToString()
{
return string.Format("Message(Header=[{0}], Data=[{1}])", ByteArrayString(Header), ByteArrayString(Data));
}
private static string ByteArrayString(byte[] bytes)
{
return string.Join(",", bytes.Select(b => b.ToString("X")));
}
public override bool Equals(object obj)
{
var other = obj as Message;
if(obj==null)
return false;
return Equals(other);
}
protected bool Equals(Message other)
{
return IsSequenceEqual(Header, other.Header)
&& IsSequenceEqual(Data, other.Data);
}
private bool IsSequenceEqual<T>(IEnumerable<T> expected, IEnumerable<T> other)
{
if(expected==null && other==null)
return true;
if(expected==null || other==null)
return false;
return expected.SequenceEqual(other);
}
public override int GetHashCode()
{
unchecked
{
return ((Header != null ? Header.GetHashCode() : 0) * 397) ^ (Data != null ? Data.GetHashCode() : 0);
}
}
}
Now that I have all the plumbing, I can focus on the actual problem.
public static IObservable<Message> MyAnswer(IObservable<byte> source)
{
return source.Publish(s =>
{
return
Observable.Defer(()=>
//Start consuming once we see a BOM
s.SkipWhile(b => b != Message.BOM)
.Scan(new Accumulator(), (acc, cur)=>acc.Accumulate(cur))
)
.TakeWhile(acc=>!acc.IsEndOfMessage())
.Where(acc=>!acc.IsBeginingOfMessage())
.Select(acc=>acc.Value())
.ToArray()
.Where(buffer=>buffer.Any())
.Select(buffer => Message.Create(buffer))
.Repeat();
});
}
public class Accumulator
{
private int _index = 0;
private byte _current =0;
private bool _isCurrentEscaped = false;
private bool _isNextEscaped = false;
public Accumulator Accumulate(byte b)
{
_index++;
_current = b;
_isCurrentEscaped = _isNextEscaped;
_isNextEscaped = (!IsHeader() && !_isCurrentEscaped && b==Message.Control);
return this;
}
public byte Value()
{
return _current;
}
private bool IsHeader()
{
return _index < 5;
}
public bool IsBeginingOfMessage()
{
return _index == 1 && _current == Message.BOM;
}
public bool IsEndOfMessage()
{
return !IsHeader()
&& _current == Message.EOM
&& !_isCurrentEscaped;
}
}
For completeness, here is the guts of #Evk's answer so you easily swap in and out implementations.
public static IObservable<Message> CurrentAnswer(IObservable<byte> source)
{
return Observable.Create<Message>(o =>
{
// some crude parsing code for the sake of example
bool nextIsEscaped = false;
bool readingHeader = false;
bool readingBody = false;
List<byte> body = new List<byte>();
List<byte> header = new List<byte>();
return source.Subscribe(b =>
{
if (b == 0x81 && !nextIsEscaped && !readingHeader)
{
// start
readingHeader = true;
readingBody = false;
nextIsEscaped = false;
}
else if (b == 0x82 && !nextIsEscaped && !readingHeader)
{
// end
readingHeader = false;
readingBody = false;
if (header.Count > 0 || body.Count > 0)
{
o.OnNext(new Message()
{
Header = header.ToArray(),
Data = body.ToArray()
});
header.Clear();
body.Clear();
}
nextIsEscaped = false;
}
else if (b == 0x1B && !nextIsEscaped && !readingHeader)
{
nextIsEscaped = true;
}
else
{
if (readingHeader)
{
header.Add(b);
if (header.Count == 3)
{
readingHeader = false;
readingBody = true;
}
}
else if (readingBody)
body.Add(b);
nextIsEscaped = false;
}
});
});
}

You can just use basic building blocks: Observable.Create and Subscribe. First let's grab some code which will help us to read stream as observable of byte[] (there are many different examples of how to do that):
static class Extensions {
public static IObservable<byte[]> AsyncRead(this Stream stream, int bufferSize) {
var buffer = new byte[bufferSize];
var asyncRead = Observable.FromAsyncPattern<byte[], int, int, int>(
stream.BeginRead,
stream.EndRead);
return Observable.While(
() => stream.CanRead,
Observable.Defer(() => asyncRead(buffer, 0, bufferSize))
.Select(readBytes => buffer.Take(readBytes).ToArray()));
}
}
Then define message class:
class Message {
public byte[] Header { get; set; }
public byte[] Body { get; set; }
}
And then put that into small console app:
static void Main(string[] args) {
// original stream
var stream = new MemoryStream(new byte[] { 0x81, 0x01,0x02,0x03,0x1B,0x1B,0x82,0x82});
// your initial IObservable<byte[]>
IObservable<byte[]> bytes = stream.AsyncRead(128); // or any other buffer size
// your IObservable<Message>
var observable = Observable.Create<Message>(observer => {
// some crude parsing code for the sake of example
bool nextIsEscaped = false;
bool readingHeader = false;
bool readingBody = false;
List<byte> body = new List<byte>();
List<byte> header = new List<byte>();
return bytes.Subscribe(buffer => {
foreach (var b in buffer) {
if (b == 0x81 && !nextIsEscaped && !readingHeader) {
// start
readingHeader = true;
readingBody = false;
nextIsEscaped = false;
}
else if (b == 0x82 && !nextIsEscaped && !readingHeader) {
// end
readingHeader = false;
readingBody = false;
if (header.Count > 0 || body.Count > 0) {
observer.OnNext(new Message() {
Header = header.ToArray(),
Body = body.ToArray()
});
header.Clear();
body.Clear();
}
nextIsEscaped = false;
}
else if (b == 0x1B && !nextIsEscaped && !readingHeader) {
nextIsEscaped = true;
}
else {
if (readingHeader) {
header.Add(b);
if (header.Count == 3) {
readingHeader = false;
readingBody = true;
}
}
else if (readingBody)
body.Add(b);
nextIsEscaped = false;
}
}
});
});
observable.Subscribe(msg =>
{
Console.WriteLine("Header: " + BitConverter.ToString(msg.Header));
Console.WriteLine("Body: " + BitConverter.ToString(msg.Body));
});
Console.ReadKey();
}

C# and USBHIDDRIVER write() always return false

Work with http://www.florian-leitner.de/index.php/2007/08/03/hid-usb-driver-library/
`USBHIDDRIVER.USBInterface usbI = new USBInterface("vid_0b6a", "pid_0022");
byte[] startCMD = new byte[8];
startCMD[7] = 4;
usbI.Connect();
usbI.write(startCMD);`
"cnnect" return true but "write' always return false, return in this place:
`Result = USBSharp.WriteFile(hidHandle, ref outputReportBuffer[0],
outputReportBuffer.Length, ref NumberOfBytesWritten, 0);
Success = (Result == 0) ? false : true;`

Extract Embedded Image Object in RTF

I have rtf documents that include an embedded object (an image). I need to extract this as an Image object (or any other usable format). I have checked out this CodeProject article but the default apps don't render it correctly (They render the 'default image' image, not the image itself), so I moved on.
Here is a sample of the RTF Code (I had to shorten it because of size):
{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 MS Sans Serif;}}
\viewkind4\uc1\pard\lang1033\f0\fs18{\object\objemb{\*\objclass Package}\objw855\objh810{\*\objdata
01050000
02000000
08000000
5061636b61676500
00000000
00000000
1f900000
02007369675f5f2e6a706700433a5c55736572735c726563657074696f6e5c4465736b746f705c
5369676e6174757265735c7369675f5f2e6a7067000000030034000000433a5c55736572735c52
45434550547e315c417070446174615c4c6f63616c5c54656d705c7369675f5f20283132292e6a
706700c18e0000ffd8ffe000104a46494600010101004800470000ffdb00430001010101010101
010101010101010101010101010101010101010101010101010101010101010101010101010101
010101010101010101010101010101010101ffdb00430101010101010101010101010101010101
010101010101010101010101010101010101010101010101010101010101010101010101010101
010101010101010101ffc0001108012c03e803012200021101031101ffc4001f00010002030002
0301000000000000000000090a07080b050602030401ffc4003f10000006030001040201030301
04070900000203040506010708090a11121314152116172223314118192532591a24576598d6d8
2933384651788497b7ffc4001a010101000301010000000000000000000000030204050106ffc4
002b11010003010100020103030402030000000002030401051112130614211522230731415124
32536162ffda000c03010002110311003f00bfc000000000000000000000000000000000000000
000000000000000000000000000000000000000000000000000000000000000000000000000000
000000000000000000000000000000000000000000000000000000000000000000000000000000
000000000000000000000000000000000000000000000000000000000000000000000000000000
000000000000000000000000000000000000000000000000000000000000000000000000000000
000000000000000000000000000000000000000000000000000000000000000000000000000000
000000000000000000000000000000000000000000000000000000000000000000000000000000
000000000000000000000000000000000000000000000000000000000000000000000000000000
000000000000000000000000000000000000000000000000000000000000000000000000000000
000000000000000000000000000000000000000000000000000000000000000000000000000000
...
005c0072006500630065007000740069006f006e005c004400650073006b0074006f0070005c00
5300690067006e006100740075007200650073005c007300690067005f005f002e006a00700067
00
01050000
00000000
}{\result{\pict\wmetafile8\picw2010\pich1905\picwgoal855\pichgoal810
0100090000033b0700000200210600000000050000000b0200000000050000000c02350038001c
000000fb02f4ff000000000000900100000001000000005365676f65205549000e0a52104c2308
00dd1900d894ef758001f3758d0e664a040000002d010000050000000902000000000500000001
02ffffff00a5000000410bc600880020002000000000002000200000000c002800000020000000
400000000100010000000000000100000000000000000000000000000000000000000000ffffff
...
0021001c001c000000fb021000070000000000bc02000000000102022253797374656d00008d0e
664a00000a0022008a0100000000ffffffff8cdd1900040000002d010100030000000000
}}}\par
}

Here is a piece of code that can extract all objects ('Package' class objects) from an RTF stream:
public static void ExtractPackageObjects(string filePath)
{
using (StreamReader sr = new StreamReader(filePath))
{
RtfReader reader = new RtfReader(sr);
IEnumerator<RtfObject> enumerator = reader.Read().GetEnumerator();
while(enumerator.MoveNext())
{
if (enumerator.Current.Text == "object")
{
if (RtfReader.MoveToNextControlWord(enumerator, "objclass"))
{
string className = RtfReader.GetNextText(enumerator);
if (className == "Package")
{
if (RtfReader.MoveToNextControlWord(enumerator, "objdata"))
{
byte[] data = RtfReader.GetNextTextAsByteArray(enumerator);
using (MemoryStream packageData = new MemoryStream())
{
RtfReader.ExtractObjectData(new MemoryStream(data), packageData);
packageData.Position = 0;
PackagedObject po = PackagedObject.Extract(packageData);
File.WriteAllBytes(po.DisplayName, po.Data);
}
}
}
}
}
}
}
}
And here are the utility classes that this code uses. There is a simple stream-based RTF parser that allows to get to the interesting control words.
There is also a utility to extract data from a serialized Object Packager instance. Object Packager is an almost 20-years ago OLE1.0 thing and the serialized binary format is not documented (to my knowledge), but it's understandable.
This works fine on your provided sample, but you may have to adapt things around.
public class RtfReader
{
public RtfReader(TextReader reader)
{
if (reader == null)
throw new ArgumentNullException("reader");
Reader = reader;
}
public TextReader Reader { get; private set; }
public IEnumerable<RtfObject> Read()
{
StringBuilder controlWord = new StringBuilder();
StringBuilder text = new StringBuilder();
Stack<RtfParseState> stack = new Stack<RtfParseState>();
RtfParseState state = RtfParseState.Group;
do
{
int i = Reader.Read();
if (i < 0)
{
if (!string.IsNullOrWhiteSpace(controlWord.ToString()))
yield return new RtfControlWord(controlWord.ToString());
if (!string.IsNullOrWhiteSpace(text.ToString()))
yield return new RtfText(text.ToString());
yield break;
}
char c = (char)i;
// noise chars
if ((c == '\r') ||
(c == '\n'))
continue;
switch (state)
{
case RtfParseState.Group:
if (c == '{')
{
stack.Push(state);
break;
}
if (c == '\\')
{
state = RtfParseState.ControlWord;
break;
}
break;
case RtfParseState.ControlWord:
if (c == '\\')
{
// another controlWord
if (!string.IsNullOrWhiteSpace(controlWord.ToString()))
{
yield return new RtfControlWord(controlWord.ToString());
controlWord.Clear();
}
break;
}
if (c == '{')
{
// a new group
state = RtfParseState.Group;
if (!string.IsNullOrWhiteSpace(controlWord.ToString()))
{
yield return new RtfControlWord(controlWord.ToString());
controlWord.Clear();
}
break;
}
if (c == '}')
{
// close group
state = stack.Count > 0 ? stack.Pop() : RtfParseState.Group;
if (!string.IsNullOrWhiteSpace(controlWord.ToString()))
{
yield return new RtfControlWord(controlWord.ToString());
controlWord.Clear();
}
break;
}
if (!Char.IsLetterOrDigit(c))
{
state = RtfParseState.Text;
text.Append(c);
if (!string.IsNullOrWhiteSpace(controlWord.ToString()))
{
yield return new RtfControlWord(controlWord.ToString());
controlWord.Clear();
}
break;
}
controlWord.Append(c);
break;
case RtfParseState.Text:
if (c == '\\')
{
state = RtfParseState.EscapedText;
break;
}
if (c == '{')
{
if (!string.IsNullOrWhiteSpace(text.ToString()))
{
yield return new RtfText(text.ToString());
text.Clear();
}
// a new group
state = RtfParseState.Group;
break;
}
if (c == '}')
{
if (!string.IsNullOrWhiteSpace(text.ToString()))
{
yield return new RtfText(text.ToString());
text.Clear();
}
// close group
state = stack.Count > 0 ? stack.Pop() : RtfParseState.Group;
break;
}
text.Append(c);
break;
case RtfParseState.EscapedText:
if ((c == '\\') || (c == '}') || (c == '{'))
{
state = RtfParseState.Text;
text.Append(c);
break;
}
// ansi character escape
if (c == '\'')
{
text.Append(FromHexa((char)Reader.Read(), (char)Reader.Read()));
break;
}
if (!string.IsNullOrWhiteSpace(text.ToString()))
{
yield return new RtfText(text.ToString());
text.Clear();
}
// in fact, it's a normal controlWord
controlWord.Append(c);
state = RtfParseState.ControlWord;
break;
}
}
while (true);
}
public static bool MoveToNextControlWord(IEnumerator<RtfObject> enumerator, string word)
{
if (enumerator == null)
throw new ArgumentNullException("enumerator");
while (enumerator.MoveNext())
{
if (enumerator.Current.Text == word)
return true;
}
return false;
}
public static string GetNextText(IEnumerator<RtfObject> enumerator)
{
if (enumerator == null)
throw new ArgumentNullException("enumerator");
while (enumerator.MoveNext())
{
RtfText text = enumerator.Current as RtfText;
if (text != null)
return text.Text;
}
return null;
}
public static byte[] GetNextTextAsByteArray(IEnumerator<RtfObject> enumerator)
{
if (enumerator == null)
throw new ArgumentNullException("enumerator");
while (enumerator.MoveNext())
{
RtfText text = enumerator.Current as RtfText;
if (text != null)
{
List<byte> bytes = new List<byte>();
for (int i = 0; i < text.Text.Length; i += 2)
{
bytes.Add((byte)FromHexa(text.Text[i], text.Text[i + 1]));
}
return bytes.ToArray();
}
}
return null;
}
// Extracts an EmbeddedObject/ObjectHeader from a stream
// see [MS -OLEDS]: Object Linking and Embedding (OLE) Data Structures for more information
// chapter 2.2: OLE1.0 Format Structures
public static void ExtractObjectData(Stream inputStream, Stream outputStream)
{
if (inputStream == null)
throw new ArgumentNullException("inputStream");
if (outputStream == null)
throw new ArgumentNullException("outputStream");
BinaryReader reader = new BinaryReader(inputStream);
reader.ReadInt32(); // OLEVersion
int formatId = reader.ReadInt32(); // FormatID
if (formatId != 2) // see 2.2.4 Object Header. 2 means EmbeddedObject
throw new NotSupportedException();
ReadLengthPrefixedAnsiString(reader); // className
ReadLengthPrefixedAnsiString(reader); // topicName
ReadLengthPrefixedAnsiString(reader); // itemName
int nativeDataSize = reader.ReadInt32();
byte[] bytes = reader.ReadBytes(nativeDataSize);
outputStream.Write(bytes, 0, bytes.Length);
}
// see chapter 2.1.4 LengthPrefixedAnsiString
private static string ReadLengthPrefixedAnsiString(BinaryReader reader)
{
int length = reader.ReadInt32();
if (length == 0)
return string.Empty;
byte[] bytes = reader.ReadBytes(length);
return Encoding.Default.GetString(bytes, 0, length - 1);
}
private enum RtfParseState
{
ControlWord,
Text,
EscapedText,
Group
}
private static char FromHexa(char hi, char lo)
{
return (char)byte.Parse(hi.ToString() + lo, NumberStyles.HexNumber);
}
}
// Utility class to parse an OLE1.0 OLEOBJECT
public class PackagedObject
{
private PackagedObject()
{
}
public string DisplayName { get; private set; }
public string IconFilePath { get; private set; }
public int IconIndex { get; private set; }
public string FilePath { get; private set; }
public byte[] Data { get; private set; }
private static string ReadAnsiString(BinaryReader reader)
{
StringBuilder sb = new StringBuilder();
do
{
byte b = reader.ReadByte();
if (b == 0)
return sb.ToString();
sb.Append((char)b);
}
while (true);
}
public static PackagedObject Extract(Stream inputStream)
{
if (inputStream == null)
throw new ArgumentNullException("inputStream");
BinaryReader reader = new BinaryReader(inputStream);
reader.ReadUInt16(); // sig
PackagedObject po = new PackagedObject();
po.DisplayName = ReadAnsiString(reader);
po.IconFilePath = ReadAnsiString(reader);
po.IconIndex = reader.ReadUInt16();
int type = reader.ReadUInt16();
if (type != 3) // 3 is file, 1 is link
throw new NotSupportedException();
reader.ReadInt32(); // nextsize
po.FilePath = ReadAnsiString(reader);
int dataSize = reader.ReadInt32();
po.Data = reader.ReadBytes(dataSize);
// note after that, there may be unicode + long path info
return po;
}
}
public class RtfObject
{
public RtfObject(string text)
{
if (text == null)
throw new ArgumentNullException("text");
Text = text.Trim();
}
public string Text { get; private set; }
}
public class RtfText : RtfObject
{
public RtfText(string text)
: base(text)
{
}
}
public class RtfControlWord : RtfObject
{
public RtfControlWord(string name)
: base(name)
{
}
}

OK, this should work for you. To demonstrate my solution, I created a WinForms project with a PictureBox whose paint event handler was mapped to the following function:
private void rtfImage_Paint(object sender, PaintEventArgs e)
{
string rtfStr = System.IO.File.ReadAllText("MySampleFile.rtf");
string imageDataHex = ExtractImgHex(rtfStr);
byte[] imageBuffer = ToBinary(imageDataHex);
Image image;
using (MemoryStream stream = new MemoryStream(imageBuffer))
{
image = Image.FromStream(stream);
}
Rectangle rect = new Rectangle(0, 0, 100, 100);
e.Graphics.DrawImage(image, rect);
}
This code relies the on the System.Drawing.Image.FromStream() method, along with two "helper" functions:
A string extractor:
string ExtractImgHex(string s)
{
// I'm sure you could use regex here, but this works.
// This assumes one picture per file; loops required otherwise
int pictTagIdx = s.IndexOf("{\\pict\\");
int startIndex = s.IndexOf(" ", pictTagIdx)+1;
int endIndex = s.IndexOf("}", startIndex);
return s.Substring(startIndex, endIndex - startIndex);
}
... and a binary converter:
public static byte[] ToBinary(string imageDataHex)
{
//this function taken entirely from:
// http://www.codeproject.com/Articles/27431/Writing-Your-Own-RTF-Converter
if (imageDataHex == null)
{
throw new ArgumentNullException("imageDataHex");
}
int hexDigits = imageDataHex.Length;
int dataSize = hexDigits / 2;
byte[] imageDataBinary = new byte[dataSize];
StringBuilder hex = new StringBuilder(2);
int dataPos = 0;
for (int i = 0; i < hexDigits; i++)
{
char c = imageDataHex[i];
if (char.IsWhiteSpace(c))
{
continue;
}
hex.Append(imageDataHex[i]);
if (hex.Length == 2)
{
imageDataBinary[dataPos] = byte.Parse(hex.ToString(), System.Globalization.NumberStyles.HexNumber);
dataPos++;
hex.Remove(0, 2);
}
}
return imageDataBinary;
}

Below code can extract all type of embedded objects. including image/docs/mails etc with original file name. And save them in a local path.
string MyDir = #"E:\temp\";
Document doc = new Document(MyDir + "Requirement#4.rtf");
NodeCollection nodeColl = doc.GetChildNodes(NodeType.Shape, true);
foreach (var node in nodeColl)
{
Shape shape1 = (Shape)node;
if (shape1.OleFormat != null)
{
shape1.OleFormat.Save(MyDir + shape1.OleFormat.SuggestedFileName + shape1.OleFormat.SuggestedExtension);
}
}