Related
I read from a stream that contains CRLF each X bytes (for me X is 2033 because the file is generated with bcp, I put X to 4 in sample code). I would like to transform the stream into another stream without this CRLF. The new stream will be deserialized as xml.
Likewise, I can do it easily and runs gracefully in this way:
using System.IO;
using System.Text;
using System.Xml.Linq;
public class CREliminator
{
public const int BcpChunkSize = 4;
public Stream Run(StreamReader reader)
{
var auxstream = new MemoryStream();
var auxwriter = new StreamWriter(auxstream);
var chunk = new char[BcpChunkSize];
do
{
var n_bytes =
reader
.ReadBlock(chunk, 0, BcpChunkSize);
auxwriter.Write(chunk[..n_bytes]);
auxwriter.Flush();
if (n_bytes == BcpChunkSize)
{
char[] chunk2 = new char[2];
n_bytes = reader.ReadBlock(chunk2, 0, 2);
}
} while (!reader.EndOfStream);
auxstream.Position = 0;
return auxstream;
}
}
public class UnitTest1
{
[Fact]
public void Test1()
{
var CRLF="\r\n";
var string_data = $"<doc{CRLF}umen{CRLF}t>A<{CRLF}/doc{CRLF}umen{CRLF}t>";
var expected = string_data.Replace(CRLF, "");
// to stream
var memory = new MemoryStream(Encoding.UTF8.GetBytes(string_data));
var data = new StreamReader(memory);
// act
var result = new CREliminator().Run(data);
// assert
var x = XDocument.Load(result);
Assert.Equal(expected, x.ToString());
}
}
But this code loads all stream in memory before to return the new stream.
My question is, how can do it in a Lazy mode? I mean, processing stream when some process is reading from new stream.
Thanks.
Only Stream.Read method needs to be implemented:
Read from source stream in chunks
Skip 2 bytes after every chunk
using System.Buffers;
using System.Text;
var sourceString = string.Concat(
Enumerable.Range(1, 10).Select(_ => "Foo \r\nBar \r\nBaz!\r\n"));
Console.WriteLine("Source: " + sourceString);
var encoding = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false);
var sourceBytes = encoding.GetBytes(sourceString);
using var stream = new MemoryStream(sourceBytes);
using var filter = new CrLfFilteringStream(stream, 4);
using var reader = new StreamReader(filter, encoding);
var res = reader.ReadToEnd();
Console.WriteLine("Result: " + res);
public class CrLfFilteringStream : Stream
{
private readonly Stream _stream;
private readonly int _chunkSize;
private readonly byte[] _chunk;
private int _chunkPosition;
private int _chunkLength;
public CrLfFilteringStream(Stream stream, int chunkSize)
{
_stream = stream ?? throw new ArgumentNullException(nameof(stream));
_chunkSize = chunkSize;
_chunkPosition = chunkSize;
_chunkLength = chunkSize;
_chunk = ArrayPool<byte>.Shared.Rent(chunkSize);
}
public override int Read(byte[] buffer, int offset, int count)
{
var toRead = count;
var bufferPosition = 0;
Span<byte> sink = stackalloc byte[2];
while (toRead > 0 && _chunkLength > 0)
{
if (_chunkPosition >= _chunkSize)
{
_chunkPosition = 0;
_chunkLength = _stream.Read(_chunk, 0, _chunkSize);
// Skip CR LF.
_stream.Read(sink);
}
var currentRead = Math.Min(_chunkLength, toRead);
Array.Copy(_chunk, _chunkPosition, buffer, bufferPosition, currentRead);
toRead -= currentRead;
bufferPosition += currentRead;
_chunkPosition += currentRead;
}
return count - toRead;
}
public override void Flush() => throw new NotSupportedException();
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
public override void SetLength(long value) => throw new NotSupportedException();
public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException();
public override bool CanRead => true;
public override bool CanSeek => false;
public override bool CanWrite => false;
public override long Length => throw new NotSupportedException();
public override long Position
{
get => throw new NotSupportedException();
set => throw new NotSupportedException();
}
protected override void Dispose(bool disposing)
{
ArrayPool<byte>.Shared.Return(_chunk);
base.Dispose(disposing);
}
}
This code rents a single chunk-sized buffer from ArrayPool and does not allocate anything else (besides CrLfFilteringStream instance).
I have been trying to get a custom audio stream to work with SharpDX.MediaFoundation.
To this end I have wrapped my audio object in a class that implements System.IO.Stream as follows:
public class AudioReaderWaveStream : System.IO.Stream
{
byte[] waveHeader = new byte[44];
AudioCore.IAudioReader reader = null;
ulong readHandle = 0xffffffff;
long readPosition = 0;
public AudioReaderWaveStream(AudioCore.CEditedAudio content)
{
reader = content as AudioCore.IAudioReader;
readHandle = reader.OpenDevice();
int sampleRate = 0;
short channels = 0;
content.GetFormat(out sampleRate, out channels);
System.IO.MemoryStream memStream = new System.IO.MemoryStream(waveHeader);
using (System.IO.BinaryWriter bw = new System.IO.BinaryWriter(memStream))
{
bw.Write("RIFF".ToCharArray());
bw.Write((Int32)Length - 8);
bw.Write("WAVE".ToCharArray());
bw.Write("fmt ".ToCharArray());
bw.Write((Int32)16);
bw.Write((Int16)3);
bw.Write((Int16)1);
bw.Write((Int32)sampleRate);
bw.Write((Int32)sampleRate * 4);
bw.Write((Int16)4);
bw.Write((Int16)32);
bw.Write("data".ToCharArray());
bw.Write((Int32)reader.GetSampleCount() * 4);
}
}
protected override void Dispose(bool disposing)
{
if (readHandle != 0xffffffff)
{
reader.CloseDevice(readHandle);
readHandle = 0xfffffffff;
}
base.Dispose(disposing);
}
~AudioReaderWaveStream()
{
Dispose();
}
public override bool CanRead
{
get
{
return true;
}
}
public override bool CanSeek
{
get
{
return true;
}
}
public override bool CanWrite
{
get
{
return false;
}
}
public override long Length
{
get
{
// Number of float samples + header of 44 bytes.
return (reader.GetSampleCount() * 4) + 44;
}
}
public override long Position
{
get
{
return readPosition;
}
set
{
readPosition = value;
}
}
public override void Flush()
{
//throw new NotImplementedException();
}
public override int Read(byte[] buffer, int offset, int count)
{
if (count <= 0)
return 0;
int retCount = count;
if (Position < 44)
{
int headerCount = count;
if ( Position + count >= 44 )
{
headerCount = 44 - (int)Position;
}
Array.Copy(waveHeader, Position, buffer, offset, headerCount);
offset += headerCount;
Position += headerCount;
count -= headerCount;
}
if (count > 0)
{
float[] readBuffer = new float[count/4];
reader.Seek(readHandle, Position - 44);
reader.ReadAudio(readHandle, readBuffer);
Array.Copy(readBuffer, 0, buffer, offset, count);
}
return retCount;
}
public override long Seek(long offset, System.IO.SeekOrigin origin)
{
if (origin == System.IO.SeekOrigin.Begin)
{
readPosition = offset;
}
else if (origin == System.IO.SeekOrigin.Current)
{
readPosition += offset;
}
else
{
readPosition = Length - offset;
}
return readPosition;
}
public override void SetLength(long value)
{
throw new NotImplementedException();
}
public override void Write(byte[] buffer, int offset, int count)
{
throw new NotImplementedException();
}
}
I then take this object and create a source resolver using it as follows:
// Create a source resolver.
SharpDX.MediaFoundation.ByteStream sdxByteStream = new ByteStream( ARWS );
SharpDX.MediaFoundation.SourceResolver resolver = new SharpDX.MediaFoundation.SourceResolver();
ComObject source = (ComObject)resolver.CreateObjectFromStream( sdxByteStream, "File.wav", SourceResolverFlags.MediaSource );
However every time I'm doing this it hangs on the CreateObjectFromStream call. I've had a look inside SharpDX to see whats going on and it seems the actual hang occurs when it makes the call to the underlying interface through CreateObjectFromByteStream. I've also looked to see what data is actually read from the byte stream. It reads the first 16 bytes which includes the 'RIFF', the RIFF size, the 'WAVE' and the 'fmt '. Then nothing else.
Has anyone got any ideas what I could be doing wrong. I've tried all sorts of combinations of the SourceResolverFlags but nothing seems to make any difference. It just hangs.
It does remind me somewhat of interthread marshalling but all the media foundation calls are made from the same thread so I don't think its that. I'm also fairly sure that MediaFoundation uses free threading so this shouldn't be a problem anyway.
Has anyone any idea what I could possibly be doing wrong?
Thanks!
Ok I have come up with a solution to this. It looks like I may be having a COM threading issue. The read happens in a thread and that thread was calling back to the main thread which the function was called from.
So I used the async version of the call and perform an Application.DoEvents() to hand across control where necessary.
Callback cb = new Callback( resolver );
IUnknown cancel = null;
resolver.BeginCreateObjectFromByteStream( sdxByteStream, "File.wav", (int)(SourceResolverFlags.MediaSource | SourceResolverFlags.ByteStream), null, out cancel, cb, null );
if ( cancel != null )
{
cancel.Dispose();
}
while( cb.MediaSource == null )
{
System.Windows.Forms.Application.DoEvents();
}
SharpDX.MediaFoundation.MediaSource mediaSource = cb.MediaSource;
I really hate COM's threading model ...
When I'm trying to write very large amount of data (list with 300 000 rows and more) to memory stream using CsvHelper, it throws the exception "System.IO.IOException: Stream was too long.".
Data class is rather big and has ~30 properties, consequently each record in the file would have ~30 columns.
This is the actual writing code where exception throws (by the way this code is based on that answer of CsvHelper lib's author):
using (var memoryStream = new MemoryStream())
{
using (var streamWriter = new StreamWriter(memoryStream, encoding ?? Encoding.ASCII))
{
var csvWriter = new CsvWriter(streamWriter, GetConfiguration(delimiter, mappingClassType, mappingActions));
csvWriter.WriteRecords(data); //data is IEnumerable<T> and has more than 300k records
streamWriter.Flush();
return memoryStream.ToArray();
}
}
Then I save the resulted bytes array into the file.
File.WriteAllBytes(filePath, resultedBytesArray);
Please note, that the same code works well when I write 100 000 records to the file (in that case the file has size about 1GB). By the way, my goal is to write more then 600 000 data records.
This is the relevant part of the stack trace related to this issue.
Stream was too long.|System.IO.IOException: Stream was too long.
at System.IO.MemoryStream.Write(Byte[] buffer, Int32 offset, Int32 count)
at System.IO.StreamWriter.Flush(Boolean flushStream, Boolean flushEncoder)
at System.IO.StreamWriter.Write(Char[] buffer, Int32 index, Int32 count)
at CsvHelper.CsvWriter.NextRecord() in C:\Users\Josh\Projects\CsvHelper\src\CsvHelper\CsvWriter.cs:line 290
at CsvHelper.CsvWriter.WriteRecords(IEnumerable records) in C:\Users\Josh\Projects\CsvHelper\src\CsvHelper\CsvWriter.cs:line 490
at FileExport.Csv.CsvDocument.Create[T](IEnumerable`1 data, String delimiter, Encoding encoding, Type mappingClassType, IDictionary`2 mappingActions) in d:\Dev\DrugDevExport\FileExport\Csv\CsvDocument.cs:line 33
As far as I'm concerned the basic way to achieve my goal and avoid that issue is to split my list of written data up on few parts and concatenate them together then, but may be is there any pretty obvious and easy solution without a significant code refactoring (like increase the default stream/buffer size, etc..)?
Also keep in mind, that I've also applied two possible solutions in order to prevent "Out Of Memory" objects exception.
got rid of 2GB limitation for objects (from here https://stackoverflow.com/a/20912869) Yes, I'm running on x64 OS with 32GB RAM.
set up x64 "Platform target" in the build settings section (from here https://stackoverflow.com/a/22592876)
Thanks in advance.
You can address this limitation of 2GB by writing your own MemoryStream :
class HugeMemoryStream : Stream
{
#region Fields
private const int PAGE_SIZE = 1024000;
private const int ALLOC_STEP = 1024;
private byte[][] _streamBuffers;
private int _pageCount = 0;
private long _allocatedBytes = 0;
private long _position = 0;
private long _length = 0;
#endregion Fields
#region Internals
private int GetPageCount(long length)
{
int pageCount = (int)(length / PAGE_SIZE) + 1;
if ((length % PAGE_SIZE) == 0)
pageCount--;
return pageCount;
}
private void ExtendPages()
{
if (_streamBuffers == null)
{
_streamBuffers = new byte[ALLOC_STEP][];
}
else
{
byte[][] streamBuffers = new byte[_streamBuffers.Length + ALLOC_STEP][];
Array.Copy(_streamBuffers, streamBuffers, _streamBuffers.Length);
_streamBuffers = streamBuffers;
}
_pageCount = _streamBuffers.Length;
}
private void AllocSpaceIfNeeded(long value)
{
if (value < 0)
throw new InvalidOperationException("AllocSpaceIfNeeded < 0");
if (value == 0)
return;
int currentPageCount = GetPageCount(_allocatedBytes);
int neededPageCount = GetPageCount(value);
while (currentPageCount < neededPageCount)
{
if (currentPageCount == _pageCount)
ExtendPages();
_streamBuffers[currentPageCount++] = new byte[PAGE_SIZE];
}
_allocatedBytes = (long)currentPageCount * PAGE_SIZE;
value = Math.Max(value, _length);
if (_position > (_length = value))
_position = _length;
}
#endregion Internals
#region Stream
public override bool CanRead => true;
public override bool CanSeek => true;
public override bool CanWrite => true;
public override long Length => _length;
public override long Position
{
get { return _position; }
set
{
if (value > _length)
throw new InvalidOperationException("Position > Length");
else if (value < 0)
throw new InvalidOperationException("Position < 0");
else
_position = value;
}
}
public override void Flush() { }
public override int Read(byte[] buffer, int offset, int count)
{
int currentPage = (int)(_position / PAGE_SIZE);
int currentOffset = (int)(_position % PAGE_SIZE);
int currentLength = PAGE_SIZE - currentOffset;
long startPosition = _position;
if (startPosition + count > _length)
count = (int)(_length - startPosition);
while (count != 0 && _position < _length)
{
if (currentLength > count)
currentLength = count;
Array.Copy(_streamBuffers[currentPage++], currentOffset, buffer, offset, currentLength);
offset += currentLength;
_position += currentLength;
count -= currentLength;
currentOffset = 0;
currentLength = PAGE_SIZE;
}
return (int)(_position - startPosition);
}
public override long Seek(long offset, SeekOrigin origin)
{
switch (origin)
{
case SeekOrigin.Begin:
break;
case SeekOrigin.Current:
offset += _position;
break;
case SeekOrigin.End:
offset = _length - offset;
break;
default:
throw new ArgumentOutOfRangeException("origin");
}
return Position = offset;
}
public override void SetLength(long value)
{
if (value < 0)
throw new InvalidOperationException("SetLength < 0");
if (value == 0)
{
_streamBuffers = null;
_allocatedBytes = _position = _length = 0;
_pageCount = 0;
return;
}
int currentPageCount = GetPageCount(_allocatedBytes);
int neededPageCount = GetPageCount(value);
// Removes unused buffers if decreasing stream length
while (currentPageCount > neededPageCount)
_streamBuffers[--currentPageCount] = null;
AllocSpaceIfNeeded(value);
if (_position > (_length = value))
_position = _length;
}
public override void Write(byte[] buffer, int offset, int count)
{
int currentPage = (int)(_position / PAGE_SIZE);
int currentOffset = (int)(_position % PAGE_SIZE);
int currentLength = PAGE_SIZE - currentOffset;
long startPosition = _position;
AllocSpaceIfNeeded(_position + count);
while (count != 0)
{
if (currentLength > count)
currentLength = count;
Array.Copy(buffer, offset, _streamBuffers[currentPage++], currentOffset, currentLength);
offset += currentLength;
_position += currentLength;
count -= currentLength;
currentOffset = 0;
currentLength = PAGE_SIZE;
}
}
#endregion Stream
}
using ICSharpCode.SharpZipLib.GZip;
using System;
using System.IO;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
// HugeMemoryStrem Test
string filename = #"gzip-filename.gz";
HugeMemoryStream ms = new HugeMemoryStream();
using (StreamWriter sw = new StreamWriter(ms, Encoding.UTF8, 16384, true))
using (FileStream fs = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read))
using (GZipInputStream gzipStream = new GZipInputStream(fs))
using (StreamReader sr = new StreamReader(gzipStream, Encoding.UTF8, false, 16384, true))
{
for (string line = sr.ReadLine(); line != null; line = sr.ReadLine())
sw.WriteLine(line);
}
ms.Seek(0, SeekOrigin.Begin);
using (StreamReader srm = new StreamReader(ms, Encoding.UTF8, false, 16384, true))
using (FileStream fs = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read))
using (GZipInputStream gzipStream = new GZipInputStream(fs))
using (StreamReader sr = new StreamReader(gzipStream, Encoding.UTF8, false, 16384, true))
{
for (string line1 = sr.ReadLine(), line2 = srm.ReadLine(); line1 != null; line1 = sr.ReadLine(), line2 = srm.ReadLine())
{
if (line1 != line2)
throw new InvalidDataException();
}
}
Many thanks Spender, like he mentioned in the comment below the question, it has been fixed by replacing MemoryStream with FileStream and writing data direct into the file.
It was absolutely useless in my case to write data to MemoryStream and then copy it again into the file without any reason. Thanks him again for opening my eyes on that fact.
My fixed code below.
using (var fileStream = File.Create(path))
{
using (var streamWriter = new StreamWriter(fileStream, encoding ?? Encoding.ASCII))
{
var csvWriter = new CsvWriter(streamWriter, GetConfiguration(delimiter, mappingClassType, mappingActions));
csvWriter.WriteRecords(data);
}
}
Now it works with any amount of input data.
I'm making a demo about sound in WindowsForm, I created 3 classes for taking data of wave file. Code is below:
RiffBlock
public class RiffBlock
{
private byte[] riffID;
private uint riffSize;
private byte[] riffFormat;
public byte[] RiffID
{
get { return riffID; }
}
public uint RiffSize
{
get { return (riffSize); }
}
public byte[] RiffFormat
{
get { return riffFormat; }
}
public RiffBlock()
{
riffID = new byte[4];
riffFormat = new byte[4];
}
public void ReadRiff(FileStream inFS)
{
inFS.Read(riffID, 0, 4);
BinaryReader binRead = new BinaryReader(inFS);
riffSize = binRead.ReadUInt32();
inFS.Read(riffFormat, 0, 4);
}
}
FormatBlock
public class FormatBlock
{
private byte[] fmtID;
private uint fmtSize;
private ushort fmtTag;
private ushort fmtChannels;
private uint fmtSamplesPerSec;
private uint fmtAverageBytesPerSec;
private ushort fmtBlockAlign;
private ushort fmtBitsPerSample;
public byte[] FmtID
{
get { return fmtID; }
}
public uint FmtSize
{
get { return fmtSize; }
}
public ushort FmtTag
{
get { return fmtTag; }
}
public ushort Channels
{
get { return fmtChannels; }
}
public uint SamplesPerSec
{
get { return fmtSamplesPerSec; }
}
public uint AverageBytesPerSec
{
get { return fmtAverageBytesPerSec; }
}
public ushort BlockAlign
{
get { return fmtBlockAlign; }
}
public ushort BitsPerSample
{
get { return fmtBitsPerSample; }
}
public FormatBlock()
{
fmtID = new byte[4];
}
public void ReadFmt(FileStream inFS)
{
inFS.Read(fmtID, 0, 4);
BinaryReader binRead = new BinaryReader(inFS);
fmtSize = binRead.ReadUInt32();
fmtTag = binRead.ReadUInt16();
fmtChannels = binRead.ReadUInt16();
fmtSamplesPerSec = binRead.ReadUInt32();
fmtAverageBytesPerSec = binRead.ReadUInt32();
fmtBlockAlign = binRead.ReadUInt16();
fmtBitsPerSample = binRead.ReadUInt16();
// This accounts for the variable format header size
// 12 bytes of Riff Header, 4 bytes for FormatId, 4 bytes for FormatSize & the Actual size of the Format Header
inFS.Seek(fmtSize + 20, System.IO.SeekOrigin.Begin);
}
}
DataBlock
public class DataBlock
{
private byte[] dataID;
private uint dataSize;
private Int16[] data;
private int dataNumSamples;
public byte[] DataID
{
get { return dataID; }
}
public uint DataSize
{
get { return dataSize; }
}
public Int16 this[int pos]
{
get { return data[pos]; }
}
public int NumSamples
{
get { return dataNumSamples; }
}
public DataBlock()
{
dataID = new byte[4];
}
public void ReadData(FileStream inFS)
{
inFS.Read(dataID, 0, 4);
BinaryReader binRead = new BinaryReader(inFS);
dataSize = binRead.ReadUInt32();
data = new Int16[dataSize];
inFS.Seek(40, SeekOrigin.Begin);
dataNumSamples = (int)(dataSize / 2);
for (int i = 0; i < dataNumSamples; i++)
{
data[i] = binRead.ReadInt16();
}
}
}
It works ok with only 16bit wave file, but when I choose a 8 bit wav file or another, the result of this command dataSize = binRead.ReadUInt32();is only 4 although the file size is big.
How I can get the data of 8bit, 24bit... wav file?
Some solutions is appreciated, thank you very much.
Your reading methodology is flawed. The length is correct but for an 8 bits per sample file you should be reading bytes not words; as it stands the data will be incorrect and the value returned by the NumSamples property will be wrong.
In my case, the sub chunk size is 1160, the number of channels is 1 (mono) and the bits per sample is 8 (byte). You will need to decode the bits per sample and adjust your reading accordingly. For the WAV file I used, your program allocated a data array the correct length but 16 bit, divided the data length by 2 and called this the number of samples (wrong, it should be 1160) and then proceeded to read 580 word values from the stream.
Edit: My ancient code will not cut it in the modern age (I seem to recall having to modify it some years ago to cope with at least one additional chunk type but the details escape me).
This is what you get; anything more and your question should read "Could someone write me a program to load WAV files", as it is, we are way beyond the original question and it is time for you to knuckle down and make it work how you need it to :-)
References:
http://www.neurophys.wisc.edu/auditory/riff-format.txt
http://www-mmsp.ece.mcgill.ca/Documents/AudioFormats/WAVE/WAVE.html
http://www.shikadi.net/moddingwikiResource_Interchange_File_Format_(RIFF)
http://soundfile.sapp.org/doc/WaveFormat/
All are very useful.
///<summary>
///Reads up to 16 bit WAV files
///</summary>
///<remarks> Things have really changed in the last 15 years</remarks>
public class RiffLoader
{
public enum RiffStatus { Unknown = 0, OK, FileError, FormatError, UnsupportedFormat };
LinkedList<Chunk> chunks = new LinkedList<Chunk>();
RiffStatus status = RiffStatus.Unknown;
List<String> errorMessages = new List<string>();
String source;
public String SourceName { get { return source; } }
public RiffStatus Status { get { return status; } }
public String[] Messages { get { return errorMessages.ToArray(); } }
enum chunkType { Unknown = 0, NoMore, Riff, Fmt, Fact, Data, Error = -1 };
static Int32 scan32bits(byte[] source, int offset = 0)
{
return source[offset] | source[offset + 1] << 8 | source[offset + 2] << 16 | source[offset + 3] << 24;
}
static Int32 scan16bits(byte[] source, int offset = 0)
{
return source[offset] | source[offset + 1] << 8;
}
static Int32 scan8bits(byte[] source, int offset = 0)
{
return source[offset];
}
abstract class Chunk
{
public chunkType Ident = chunkType.Unknown;
public int ByteCount = 0;
}
class RiffChunk : Chunk
{
public RiffChunk(int count)
{
this.Ident = chunkType.Riff;
this.ByteCount = count;
}
}
class FmtChunk : Chunk
{
int formatCode = 0;
int channels = 0;
int samplesPerSec = 0;
int avgBytesPerSec = 0;
int blockAlign = 0;
int bitsPerSample = 0;
int significantBits = 0;
public int Format { get { return formatCode; } }
public int Channels { get { return channels; } }
public int BlockAlign { get { return blockAlign; } }
public int BytesPerSample { get { return bitsPerSample / 8 + ((bitsPerSample % 8) > 0 ? 1 : 0); } }
public int BitsPerSample
{
get
{
if (significantBits > 0)
return significantBits;
return bitsPerSample;
}
}
public FmtChunk(byte[] buffer) : base()
{
int size = buffer.Length;
// if the length is 18 then buffer 16,17 should be 00 00 (I don't bother checking)
if (size != 16 && size != 18 && size != 40)
return;
formatCode = scan16bits(buffer, 0);
channels = scan16bits(buffer, 2);
samplesPerSec = scan32bits(buffer, 4);
avgBytesPerSec = scan32bits(buffer, 8);
blockAlign = scan16bits(buffer, 12);
bitsPerSample = scan16bits(buffer, 14);
if (formatCode == 0xfffe) // EXTENSIBLE
{
if (size != 40)
return;
significantBits = scan16bits(buffer, 18);
// skiping speaker map
formatCode = scan16bits(buffer, 24); // first two bytes of the GUID
// the rest of the GUID is fixed, decode it and check it if you wish
}
this.Ident = chunkType.Fmt;
this.ByteCount = size;
}
}
class DataChunk : Chunk
{
byte[] samples = null;
///<summary>
///Create a data chunk
///<para>
///The supplied buffer must be correctly sized with zero offset and must be purely for this class
///</para>
///<summary>
///<param name="buffer">source array</param>
public DataChunk(byte[] buffer)
{
this.Ident = chunkType.Data;
this.ByteCount = buffer.Length;
samples = buffer;
}
public enum SampleStatus { OK, Duff }
public class Samples
{
public SampleStatus Status = SampleStatus.Duff;
public List<int[]> Channels = new List<int[]>();
#if false // debugger helper method
/*
** Change #if false to #if true to include this
** Break at end of GetSamples on "return retval"
** open immediate window and type retval.DumpLast(16)
** look in output window for dump of last 16 entries
*/
public int DumpLast(int count)
{
for (int i = Channels[0].Length - count; i < Channels[0].Length; i++)
Console.WriteLine(String.Format("{0:X4} {1:X4},{2:X4}", i, Channels[0][i], Channels[1][i]));
return 0;
}
#endif
}
/*
** Return the decoded samples
*/
public Samples GetSamples(FmtChunk format)
{
Samples retval = new Samples();
int samplesPerChannel = this.ByteCount / (format.BytesPerSample * format.Channels);
int mask = 0, sign=0;
int [][] samples = new int [format.Channels][];
for (int c = 0; c < format.Channels; c++)
samples[c] = new int[samplesPerChannel];
if (format.BitsPerSample >= 8 && format.BitsPerSample <= 16) // 24+ is left as an excercise
{
sign = (int)Math.Floor(Math.Pow(2, format.BitsPerSample - 1));
mask = (int)Math.Floor(Math.Pow(2, format.BitsPerSample)) - 1;
int offset = 0, index = 0;
int s = 0;
while (index < samplesPerChannel)
{
for (int c = 0; c < format.Channels; c++)
{
switch (format.BytesPerSample)
{
case 1:
s = scan8bits(this.samples, offset) & mask;
break;
case 2:
s = scan16bits(this.samples, offset) & mask;
break;
}
// sign extend the data to Int32
samples[c][index] = s | ((s & sign) != 0 ? ~mask : 0);
offset += format.BytesPerSample;
}
++index;
}
retval.Channels = new List<int[]>(samples);
retval.Status = SampleStatus.OK;
}
return retval;
}
}
class FactChunk : Chunk
{
int samplesPerChannel;
public int SamplesPerChannel { get { return samplesPerChannel; } }
public FactChunk(byte[] buffer)
{
this.Ident = chunkType.Fact;
this.ByteCount = buffer.Length;
if (buffer.Length >= 4)
samplesPerChannel = scan32bits(buffer);
}
}
class DummyChunk : Chunk
{
public DummyChunk(int size, chunkType type = chunkType.Unknown)
{
this.Ident = type;
this.ByteCount = size;
}
}
public RiffLoader(String fileName)
{
source = fileName;
try
{
using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
using (BinaryReader reader = new BinaryReader(fs))
{
Chunk c = getChunk(fs, reader);
if (c.Ident != chunkType.Riff)
{
status = RiffStatus.FileError;
errorMessages.Add(String.Format("Error loading \"{0}\": No valid header"));
}
chunks.AddLast(c);
c = getChunk(fs, reader);
if (c.Ident != chunkType.Fmt)
{
status = RiffStatus.FileError;
errorMessages.Add(String.Format("Error loading \"{0}\": No format chunk"));
}
chunks.AddLast(c);
/*
** From now on we just keep scanning to the end of the file
*/
while (fs.Position < fs.Length)
{
c = getChunk(fs, reader);
switch (c.Ident)
{
case chunkType.Fact:
case chunkType.Data:
chunks.AddLast(c);
break;
case chunkType.Unknown:
break; // skip it - don't care what it is
}
}
FmtChunk format = null;
foreach (var chunk in chunks)
{
switch(chunk.Ident)
{
case chunkType.Fmt:
format = chunk as FmtChunk;
break;
case chunkType.Data:
if (format != null)
{
DataChunk dc = chunk as DataChunk;
var x = dc.GetSamples(format);
}
break;
}
}
}
}
catch (Exception e)
{
status = RiffStatus.FileError;
errorMessages.Add(String.Format("Error loading \"{0}\": {1}", e.Message));
}
}
/*
** Get a chunk of data from the file - knows nothing of the internal format of the chunk.
*/
Chunk getChunk(FileStream stream, BinaryReader reader)
{
byte[] buffer;
int size;
buffer = reader.ReadBytes(8);
if (buffer.Length == 8)
{
String prefix = new String(Encoding.ASCII.GetChars(buffer, 0, 4));
size = scan32bits(buffer, 4);
if (size + stream.Position <= stream.Length) // skip if there isn't enough data
{
if (String.Compare(prefix, "RIFF") == 0)
{
/*
** only "WAVE" type is acceptable
**
** Don't read size bytes or the entire file will end up in the RIFF chunk
*/
if (size >= 4)
{
buffer = reader.ReadBytes(4);
String ident = new String(Encoding.ASCII.GetChars(buffer, 0, 4));
if (String.CompareOrdinal(ident, "WAVE") == 0)
return new RiffChunk(size - 4);
}
}
else if (String.Compare(prefix, "fmt ") == 0)
{
if (size >= 16)
{
buffer = reader.ReadBytes(size);
if (buffer.Length == size)
return new FmtChunk(buffer);
}
}
else if (String.Compare(prefix, "fact") == 0)
{
if (size >= 4)
{
buffer = reader.ReadBytes(4);
if (buffer.Length == size)
return new FactChunk(buffer);
}
}
else if (String.Compare(prefix, "data") == 0)
{
// assume that there has to be data
if (size > 0)
{
buffer = reader.ReadBytes(size);
if ((size & 1) != 0) // odd length?
{
if (stream.Position < stream.Length)
reader.ReadByte();
else
size = -1; // force an error - there should be a pad byte
}
if (buffer.Length == size)
return new DataChunk(buffer);
}
}
else
{
/*
** there are a number of weird and wonderful block types - assume there has to be data
*/
if (size > 0)
{
buffer = reader.ReadBytes(size);
if ((size & 1) != 0) // odd length?
{
if (stream.Position < stream.Length)
reader.ReadByte();
else
size = -1; // force an error - there should be a pad byte
}
if (buffer.Length == size)
{
DummyChunk skip = new DummyChunk(size);
return skip;
}
}
}
}
}
return new DummyChunk(0, chunkType.Error);
}
}
You will need to add properties as required and code to navigate the returned linked list. Particular properties you may need are the sample rates in the format block, assuming you are going to process the data in some way.
Adding 24 to 32 bits is simple, if you need to go beyond 32 bits you will have to switch to int64's.
I have tested it with some good samples from http://www-mmsp.ece.mcgill.ca/Documents/AudioFormats/WAVE/Samples.html, as well as your 8 bit file, and it decodes OK and seems to have the correct data.
You should be more than capable of making it work now, good luck.
Edit (again, like the proverbial dog...):
Of course, I should have sign extended the value, so DataChunk.GetSamples() now does that. Looking at the Codeproject files (it isn't the greatest code by the way, but the guy does say that he is only just learning C#, so fair play for tackling a graphical user control) it is obvious that the data is signed. It's a shame that he didn't standardise his source to be an array of Int32's, then it wouldn't matter how many bits the WAV was encoded in.
You could use the Naudio library:
https://naudio.codeplex.com/ (take a look at their website, there are quite a lot of tutorials).
Hope this helps :).
Guys, I'm trying to implement a PBKDF2 function in C# that creates a WPA Shared key. I've found some here: http://msdn.microsoft.com/en-us/magazine/cc163913.aspx that seems to produce a valid result, but it's one byte too short... and the wrong PSK value.
To test the output, I am comparing it to this: http://www.xs4all.nl/~rjoris/wpapsk.html or http://anandam.name/pbkdf2/
I did find one way of getting this to work with a built in library to C# called Rfc2898DeriveBytes. Using this, I get a valid output using:
Rfc2898DeriveBytes k3 = new Rfc2898DeriveBytes(pwd1, salt1, 4096);
byte[] answers = k3.GetBytes(32);
Now, the one limitation I have using Rfc2898DeriveBytes is the "salt" must be 8 octets long. If it is shorter, the Rfc2898DeriveBytes throws an exception. I was thinking all I had to do was pad the salt (if it was shorter) to 8 bytes, and I'd be good. But NO! I've tried pretty much every combination of padding with a shorter salt, but I cannot duplicate the results I get from those two websites above.
So bottom line is, does this mean the Rfc2898DeriveBytes just simply won't work with a source salt shorter than 8 bytes? If so, does anyone know of any C# code I could use that implements PBKDF2 for WPA Preshared key?
Here is an implementation that does not require the 8 byte salt.
You can calculate a WPA key as follows:
Rfc2898DeriveBytes rfc2898 = new Rfc2898DeriveBytes(passphrase, Encoding.UTF8.GetBytes(name), 4096);
key = rfc2898.GetBytes(32);
public class Rfc2898DeriveBytes : DeriveBytes
{
const int BlockSize = 20;
uint block;
byte[] buffer;
int endIndex;
readonly HMACSHA1 hmacsha1;
uint iterations;
byte[] salt;
int startIndex;
public Rfc2898DeriveBytes(string password, int saltSize)
: this(password, saltSize, 1000)
{
}
public Rfc2898DeriveBytes(string password, byte[] salt)
: this(password, salt, 1000)
{
}
public Rfc2898DeriveBytes(string password, int saltSize, int iterations)
{
if (saltSize < 0)
{
throw new ArgumentOutOfRangeException("saltSize");
}
byte[] data = new byte[saltSize];
new RNGCryptoServiceProvider().GetBytes(data);
Salt = data;
IterationCount = iterations;
hmacsha1 = new HMACSHA1(new UTF8Encoding(false).GetBytes(password));
Initialize();
}
public Rfc2898DeriveBytes(string password, byte[] salt, int iterations) : this(new UTF8Encoding(false).GetBytes(password), salt, iterations)
{
}
public Rfc2898DeriveBytes(byte[] password, byte[] salt, int iterations)
{
Salt = salt;
IterationCount = iterations;
hmacsha1 = new HMACSHA1(password);
Initialize();
}
static byte[] Int(uint i)
{
byte[] bytes = BitConverter.GetBytes(i);
byte[] buffer2 = new byte[] {bytes[3], bytes[2], bytes[1], bytes[0]};
if (!BitConverter.IsLittleEndian)
{
return bytes;
}
return buffer2;
}
byte[] DeriveKey()
{
byte[] inputBuffer = Int(block);
hmacsha1.TransformBlock(salt, 0, salt.Length, salt, 0);
hmacsha1.TransformFinalBlock(inputBuffer, 0, inputBuffer.Length);
byte[] hash = hmacsha1.Hash;
hmacsha1.Initialize();
byte[] buffer3 = hash;
for (int i = 2; i <= iterations; i++)
{
hash = hmacsha1.ComputeHash(hash);
for (int j = 0; j < BlockSize; j++)
{
buffer3[j] = (byte) (buffer3[j] ^ hash[j]);
}
}
block++;
return buffer3;
}
public override byte[] GetBytes(int bytesToGet)
{
if (bytesToGet <= 0)
{
throw new ArgumentOutOfRangeException("bytesToGet");
}
byte[] dst = new byte[bytesToGet];
int dstOffset = 0;
int count = endIndex - startIndex;
if (count > 0)
{
if (bytesToGet < count)
{
Buffer.BlockCopy(buffer, startIndex, dst, 0, bytesToGet);
startIndex += bytesToGet;
return dst;
}
Buffer.BlockCopy(buffer, startIndex, dst, 0, count);
startIndex = endIndex = 0;
dstOffset += count;
}
while (dstOffset < bytesToGet)
{
byte[] src = DeriveKey();
int num3 = bytesToGet - dstOffset;
if (num3 > BlockSize)
{
Buffer.BlockCopy(src, 0, dst, dstOffset, BlockSize);
dstOffset += BlockSize;
}
else
{
Buffer.BlockCopy(src, 0, dst, dstOffset, num3);
dstOffset += num3;
Buffer.BlockCopy(src, num3, buffer, startIndex, BlockSize - num3);
endIndex += BlockSize - num3;
return dst;
}
}
return dst;
}
void Initialize()
{
if (buffer != null)
{
Array.Clear(buffer, 0, buffer.Length);
}
buffer = new byte[BlockSize];
block = 1;
startIndex = endIndex = 0;
}
public override void Reset()
{
Initialize();
}
public int IterationCount
{
get
{
return (int) iterations;
}
set
{
if (value <= 0)
{
throw new ArgumentOutOfRangeException("value");
}
iterations = (uint) value;
Initialize();
}
}
public byte[] Salt
{
get
{
return (byte[]) salt.Clone();
}
set
{
if (value == null)
{
throw new ArgumentNullException("value");
}
salt = (byte[]) value.Clone();
Initialize();
}
}
}
I get matching results when comparing key-derivation from .NET's Rfc2898DeriveBytes and Anandam's PBKDF2 Javascript implementation.
I put together an example of packaging SlowAES and Anandam's PBKDF2 into Windows Script Components. Using this implementation shows good interop with the .NET RijndaelManaged class and the Rfc2898DeriveBytes class.
See also:
AES in Javascript
Getting SlowAES and RijndaelManaged to play together
All of these go further than what you are asking for. They all show interop of the AES encryption. But to get interop on encryption, it is a necessary pre-requisite to have interop (or matching outputs) on the password-based key derivation.
Looking at the Microsoft link, I made some changes in order to make the PMK the same as those discovered in the links you put forward.
Change the SHA algorithm from SHA256Managed to SHA1Managed for the inner and outer hash.
Change HASH_SIZE_IN_BYTES to equal 20 rather than 34.
This produces the correct WPA key.
I know it's a bit late coming, but I've only just started looking for this sort of informatin and thought I could help others out. If anyone does read this post, any ideas on the PRF function and how to do it within C#?
This expands on Dodgyrabbit's answer and his code helped to fix mine as I developed this. This generic class can use any HMAC-derived class in C#. This is .NET 4 because of the parameters with default values, but if those were changed then this should work down to .NET 2, but I haven't tested that. USE AT YOUR OWN RISK.
I have also posted this on my blog, The Albequerque Left Turn, today.
using System;
using System.Text;
using System.Security.Cryptography;
namespace System.Security.Cryptography
{
//Generic PBKDF2 Class that can use any HMAC algorithm derived from the
// System.Security.Cryptography.HMAC abstract class
// PER SPEC RFC2898 with help from user Dodgyrabbit on StackExchange
// http://stackoverflow.com/questions/1046599/pbkdf2-implementation-in-c-sharp-with-rfc2898derivebytes
// the use of default values for parameters in the functions puts this at .NET 4
// if you remove those defaults and create the required constructors, you should be able to drop to .NET 2
// USE AT YOUR OWN RISK! I HAVE TESTED THIS AGAINST PUBLIC TEST VECTORS, BUT YOU SHOULD
// HAVE YOUR CODE PEER-REVIEWED AND SHOULD FOLLOW BEST PRACTICES WHEN USING CRYPTO-ANYTHING!
// NO WARRANTY IMPLIED OR EXPRESSED, YOU ARE ON YOUR OWN!
// PUBLIC DOMAIN! NO COPYRIGHT INTENDED OR RESERVED!
//constrain T to be any class that derives from HMAC, and that exposes a new() constructor
public class PBKDF2<T>: DeriveBytes where T : HMAC, new()
{
//Internal variables and public properties
private int _blockSize = -1; // the byte width of the output of the HMAC algorithm
byte[] _P = null;
int _C = 0;
private T _hmac;
byte[] _S = null;
// if you called the initializer/constructor specifying a salt size,
// you will need this property to GET the salt after it was created from the crypto rng!
// GET THIS BEFORE CALLING GETBYTES()! OBJECT WILL BE RESET AFTER GETBYTES() AND
// SALT WILL BE LOST!!
public byte[] Salt { get { return (byte[])_S.Clone(); } }
// Constructors
public PBKDF2(string Password, byte[] Salt, int IterationCount = 1000)
{ Initialize(Password, Salt, IterationCount); }
public PBKDF2(byte[] Password, byte[] Salt, int IterationCount = 1000)
{ Initialize(Password, Salt, IterationCount); }
public PBKDF2(string Password, int SizeOfSaltInBytes, int IterationCount = 1000)
{ Initialize(Password, SizeOfSaltInBytes, IterationCount);}
public PBKDF2(byte[] Password, int SizeOfSaltInBytes, int IterationCount = 1000)
{ Initialize(Password, SizeOfSaltInBytes, IterationCount);}
//All Construtors call the corresponding Initialize methods
public void Initialize(string Password, byte[] Salt, int IterationCount = 1000)
{
if (string.IsNullOrWhiteSpace(Password))
throw new ArgumentException("Password must contain meaningful characters and not be null.", "Password");
if (IterationCount < 1)
throw new ArgumentOutOfRangeException("IterationCount");
Initialize(new UTF8Encoding(false).GetBytes(Password), Salt, IterationCount);
}
public void Initialize(byte[] Password, byte[] Salt, int IterationCount = 1000)
{
//all Constructors/Initializers eventually lead to this one which does all the "important" work
if (Password == null || Password.Length == 0)
throw new ArgumentException("Password cannot be null or empty.", "Password");
if (Salt == null)
Salt = new byte[0];
if (IterationCount < 1)
throw new ArgumentOutOfRangeException("IterationCount");
_P = (byte[])Password.Clone();
_S = (byte[])Salt.Clone();
_C = IterationCount;
//determine _blockSize
_hmac = new T();
_hmac.Key = new byte[] { 0 };
byte[] test = _hmac.ComputeHash(new byte[] { 0 });
_blockSize = test.Length;
}
public void Initialize(string Password, int SizeOfSaltInBytes, int IterationCount = 1000)
{
if (string.IsNullOrWhiteSpace(Password))
throw new ArgumentException("Password must contain meaningful characters and not be null.", "Password");
if (IterationCount < 1)
throw new ArgumentOutOfRangeException("IterationCount");
Initialize(new UTF8Encoding(false).GetBytes(Password), SizeOfSaltInBytes, IterationCount);
}
public void Initialize(byte[] Password, int SizeOfSaltInBytes, int IterationCount = 1000)
{
if (Password == null || Password.Length == 0)
throw new ArgumentException("Password cannot be null or empty.", "Password");
if (SizeOfSaltInBytes < 0)
throw new ArgumentOutOfRangeException("SizeOfSaltInBytes");
if (IterationCount < 1)
throw new ArgumentOutOfRangeException("IterationCount");
// You didn't specify a salt, so I'm going to create one for you of the specific byte length
byte[] data = new byte[SizeOfSaltInBytes];
RNGCryptoServiceProvider rng = new RNGCryptoServiceProvider();
rng.GetBytes(data);
// and then finish initializing...
// Get the salt from the Salt parameter BEFORE calling GetBytes()!!!!!!!!!!!
Initialize(Password, data, IterationCount);
}
~PBKDF2()
{
//*DOOT* clean up in aisle 5! *KEKERKCRACKLE*
this.Reset();
}
// required by the Derive Bytes class/interface
// this is where you request your output bytes after Initialize
// state of class Reset after use!
public override byte[] GetBytes(int ByteCount)
{
if (_S == null || _P == null)
throw new InvalidOperationException("Object not Initialized!");
if (ByteCount < 1)// || ByteCount > uint.MaxValue * blockSize)
throw new ArgumentOutOfRangeException("ByteCount");
int totalBlocks = (int)Math.Ceiling((decimal)ByteCount / _blockSize);
int partialBlock = (int)(ByteCount % _blockSize);
byte[] result = new byte[ByteCount];
byte[] buffer = null;
// I'm using TT here instead of T from the spec because I don't want to confuse it with
// the generic object T
for (int TT = 1; TT <= totalBlocks; TT++)
{
// run the F function with the _C number of iterations for block number TT
buffer = _F((uint)TT);
//IF we're not at the last block requested
//OR the last block requested is whole (not partial)
// then take everything from the result of F for this block number TT
//ELSE only take the needed bytes from F
if (TT != totalBlocks || (TT == totalBlocks && partialBlock == 0))
Buffer.BlockCopy(buffer, 0, result, _blockSize * (TT - 1), _blockSize);
else
Buffer.BlockCopy(buffer, 0, result, _blockSize * (TT - 1), partialBlock);
}
this.Reset(); // force cleanup after every use! Cannot be reused!
return result;
}
// required by the Derive Bytes class/interface
public override void Reset()
{
_C = 0;
_P.Initialize(); // the compiler might optimize this line out! :(
_P = null;
_S.Initialize(); // the compiler might optimize this line out! :(
_S = null;
if (_hmac != null)
_hmac.Clear();
_blockSize = -1;
}
// the core function of the PBKDF which does all the iterations
// per the spec section 5.2 step 3
private byte[] _F(uint I)
{
//NOTE: SPEC IS MISLEADING!!!
//THE HMAC FUNCTIONS ARE KEYED BY THE PASSWORD! NEVER THE SALT!
byte[] bufferU = null;
byte[] bufferOut = null;
byte[] _int = PBKDF2<T>.IntToBytes(I);
_hmac = new T();
_hmac.Key = (_P); // KEY BY THE PASSWORD!
_hmac.TransformBlock(_S, 0, _S.Length, _S, 0);
_hmac.TransformFinalBlock(_int, 0, _int.Length);
bufferU = _hmac.Hash;
bufferOut = (byte[])bufferU.Clone();
for (int c = 1; c < _C; c++)
{
_hmac.Initialize();
_hmac.Key = _P; // KEY BY THE PASSWORD!
bufferU = _hmac.ComputeHash(bufferU);
_Xor(ref bufferOut, bufferU);
}
return bufferOut;
}
// XOR one array of bytes into another (which is passed by reference)
// this is the equiv of data ^= newData;
private void _Xor(ref byte[] data, byte[] newData)
{
for (int i = data.GetLowerBound(0); i <= data.GetUpperBound(0); i++)
data[i] ^= newData[i];
}
// convert an unsigned int into an array of bytes BIG ENDIEN
// per the spec section 5.2 step 3
static internal byte[] IntToBytes(uint i)
{
byte[] bytes = BitConverter.GetBytes(i);
if (!BitConverter.IsLittleEndian)
{
return bytes;
}
else
{
Array.Reverse(bytes);
return bytes;
}
}
}
}