We Keep Coding

How to convert 16Bit byte array to audio clip data correctly?

I work with Media Foundataion and what I need to do is convert sound sample frame from byte to audio float data. In order to do it I use such method (that I found somewhere at google):
private static float[] Convert16BitByteArrayToAudioClipData(byte[] source, int headerOffset, int dataSize)
{
int wavSize = BitConverter.ToInt32(source, headerOffset);
headerOffset += sizeof(int);
Debug.AssertFormat(wavSize > 0 && wavSize == dataSize, "Failed to get valid 16-bit wav size: {0} from data bytes: {1} at offset: {2}", wavSize, dataSize, headerOffset);
int x = sizeof(Int16); // block size = 2
int convertedSize = wavSize / x;
float[] data = new float[convertedSize];
Int16 maxValue = Int16.MaxValue;
int i = 0;
while (i < convertedSize)
{
int offset = i * x + headerOffset;
data[i] = (float)BitConverter.ToInt16(source, offset) / maxValue;
++i;
}
Debug.AssertFormat(data.Length == convertedSize, "AudioClip .wav data is wrong size: {0} == {1}", data.Length, convertedSize);
return data;
}
I use it like this :
...
byte[] source = ...; // lenght 43776
... = Convert16BitByteArrayToAudioClipData(source , 0, 0);
...
Looks like this method works wrong, because if I pass an array with size 43776 as a result in while loop at index i = 21886 offset value will be offset = 43776 it lead to exception at this next method
data[i] = (float)BitConverter.ToInt16(source /*43776*/, offset /*43776*/) / maxValue;
because this values could not be the same.
Question is - how to fix this method? Or maybe someone can advice what to use instead?
EDIT
private static float[] Convert16BitByteArrayToAudioClipData(byte[] source)
{
float[] data = new float[source.Length];
for (int i = 0; i < source.Length; i++)
{
data[i] = (float) source[i];
}
return data;
}

Integers need to become -1..+1 floating point values
private static float[] Convert16BitByteArrayToAudioClipData(byte[] source)
{
float[] data = new float[source.Length];
for (int i = 0; i < source.Length; i++)
{
data[i] = ((float) source[i] / Int16.MaxValue); // <<---
}
return data;
}

Eventually I did it this way:
public static float[] Convert16BitByteArrayToAudioClipData(byte[] source)
{
int x = sizeof(Int16);
int convertedSize = source.Length / x;
float[] data = new float[convertedSize];
Int16 maxValue = Int16.MaxValue;
for (int i = 0; i < convertedSize; i++)
{
int offset = i * x;
data[i] = (float)BitConverter.ToInt16(source, offset) / maxValue;
++i;
}
return data;
}

Buffer.BlockCopy Array 2d c++ to C# shared memory

I write a c++ program that send via memorystream a array multi to c# aplication. But i dont know hot to use BlockCopy to array multi:
this is my program c++ that send array multi
struct Pair {
std::pair<int, int> players;
};
struct Pair* p;
HANDLE handle;
float dataSend[70];
bool startShare()
{
try
{
handle = CreateFileMappingW(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, 0, sizeof(Pair), L"DataSend");
p = (struct Pair*) MapViewOfFile(handle, FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, sizeof(Pair));
return true;
}
catch (...)
{
return false;
}
}
for (int i = 0; i < 70; i++)
{
int r1 = rand() % 10 + 1;
int r2 = rand() % 10 + 1;
p->players = { r1,r2 };
}
example aoutput:
players 0 - [10][2]
players 1 - [100][22]
players 2 - [1][26]
players 3 - [50][211]
players 4 - [32][23]
my c# program read:
public static int[,] data = new int[70, 1];
public static MemoryMappedFile mmf;
public static MemoryMappedViewStream mmfvs;
static public bool MemOpen()
{
try
{
mmf = MemoryMappedFile.OpenExisting("DataSend");
mmfvs = mmf.CreateViewStream();
return true;
}
catch
{
return false;
}
}
// here need be somethings like byte[,] bPosition = new byte[70,1];
byte[] bPosition = new byte[70];
mmfvs.Read(bPosition, 0, 100);
Buffer.BlockCopy(bPosition, 0, data, 0, bPosition.Length);
for (int i = 0; i< data.Length; i++)
{
for(int j=0;j<1;j++)
{
Console.WriteLine(data[i][j]);
}
}
example receive :
data 0 - [10][2]
data 1 - [100][22]
data 2 - [1][26]
data 3 - [50][211]
data 4 - [32][23]
I write a c++ program that send via memorystream a array multi to c# aplication. But i dont know hot to use BlockCopy to array multi:
this is my program c++ that send array multi

Try following :
MemoryStream stream = new MemoryStream();
byte[] buffer = new byte[70 * 2 * sizeof(int)];
stream.Read(buffer, 0, 70 * 2 * sizeof(int));
for (int i = 0; i< 70; i++)
{
for (int j = 0; j < 2; j++)
{
Console.WriteLine(BitConverter.ToUInt32(buffer,(i * 2 * sizeof(int)) + (j * sizeof(int))));
}
}
IntPtr bufferPtr = Marshal.AllocHGlobal(BUFFER_SIZE);
//you need to allocate memory before calling the routing
byte[] buffer = new byte[BUFFER_SIZE];
Marshal.Copy(bufferPtr, buffer, 0, BUFFER_SIZE);
for (int i = 0; i < 70; i++)
{
for (int j = 0; j < 2; j++)
{
Console.WriteLine(BitConverter.ToUInt32(buffer, (i * 2 * sizeof(int)) + (j * sizeof(int))));
}
}
//method 2
int[,] array = new int[40, 2];
IntPtr bufferPtr2 = Marshal.AllocHGlobal(Marshal.SizeOf(array));
//call routine then you code below to fill managed memory.
Marshal.PtrToStructure(bufferPtr2, array);

opus and NAudio streaming out of sync

I am adding voip in the game and since Unity's Microphone class is not supported in Web_GL and is already slow and gives floats instead of bytes. Now some people suggested me to use codec i.e Opus and then I found its wrapper along with its demo which used NAudio, well I was fairly happy with it, it was using some extra loops which after removing also gave the same result but anyway it also gave 4000 bytes with 48k sample rate which I reduced to 8k and max buffer size to 350. Here's the code for that script
private void Start()
{
//StartEncoding();
UnityEditor.EditorApplication.playmodeStateChanged = PlayModeStateChangedHandler;
}
private void PlayModeStateChangedHandler()
{
if (UnityEditor.EditorApplication.isPaused)
{
StopEncoding();
}
}
public void StartGame()
{
StartEncoding();
}
private void StartEncoding()
{
_client = FindObjectOfType<Client>();
_client.AudioReceivers += UpdateAudioOutput;
_startTime = DateTime.Now;
_bytesSent = 0;
_segmentFrames = 160;
_encoder = OpusEncoder.Create(8000, 1, FragLabs.Audio.Codecs.Opus.Application.Voip);
_encoder.MaxDataBytes = 350;
_encoder.Bitrate = 4000;
_decoder = OpusDecoder.Create(8000, 1);
_decoder.MaxDataBytes = 175;
_bytesPerSegment = _encoder.FrameByteCount(_segmentFrames);
_waveIn = new WaveIn(WaveCallbackInfo.FunctionCallback());
_waveIn.BufferMilliseconds = 50;
_waveIn.DeviceNumber = 0;
_waveIn.DataAvailable += _waveIn_DataAvailable;
_waveIn.WaveFormat = new WaveFormat(8000, 16, 1);
_playBuffer = new BufferedWaveProvider(new WaveFormat(8000, 16, 1));
_playBuffer.DiscardOnBufferOverflow = true;
_waveOut = new WaveOut(WaveCallbackInfo.FunctionCallback());
_waveOut.DeviceNumber = 0;
_waveOut.Init(_playBuffer);
_waveOut.Play();
_waveIn.StartRecording();
if (_timer == null)
{
_timer = new Timer();
_timer.Interval = 1000;
_timer.Elapsed += _timer_Tick;
}
_timer.Start();
}
private void _timer_Tick(object sender, EventArgs e)
{
var timeDiff = DateTime.Now - _startTime;
var bytesPerSecond = _bytesSent / timeDiff.TotalSeconds;
}
byte[] _notEncodedBuffer = new byte[0];
private void _waveIn_DataAvailable(object sender, WaveInEventArgs e)
{
byte[] soundBuffer = new byte[e.BytesRecorded + _notEncodedBuffer.Length];
for (int i = 0; i < _notEncodedBuffer.Length; i++)
soundBuffer[i] = _notEncodedBuffer[i];
for (int i = 0; i < e.BytesRecorded; i++)
soundBuffer[i + _notEncodedBuffer.Length] = e.Buffer[i];
int byteCap = _bytesPerSegment;
int segmentCount = (int)Math.Floor((decimal)soundBuffer.Length / byteCap);
int segmentsEnd = segmentCount * byteCap;
int notEncodedCount = soundBuffer.Length - segmentsEnd;
_notEncodedBuffer = new byte[notEncodedCount];
for (int i = 0; i < notEncodedCount; i++)
{
_notEncodedBuffer[i] = soundBuffer[segmentsEnd + i];
}
for (int i = 0; i < segmentCount; i++)
{
byte[] segment = new byte[byteCap];
for (int j = 0; j < segment.Length; j++)
segment[j] = soundBuffer[(i * byteCap) + j];
int len;
byte[] buff = _encoder.Encode(segment, segment.Length, out len);
SendToServer(buff, len);
}
}
public void UpdateAudioOutput(byte[] ba, int len)
{
int outlen = len;
byte[] buff = new byte[len];
buff = _decoder.Decode(ba, outlen, out outlen);
_playBuffer.AddSamples(buff, 0, outlen);
}
private void SendToServer(byte[] EncodedAudio, int Length)
{
print("SENDING AUDIO");
//print("audio length : " + EncodedAudio.Length);
_client.Send(EncodedAudio, Length);
//UpdateAudioOutput(EncodedAudio, Length);
}
private void StopEncoding()
{
_timer.Stop();
_waveIn.StopRecording();
_waveIn.Dispose();
_waveIn = null;
_waveOut.Stop();
_waveOut.Dispose();
_waveOut = null;
_playBuffer = null;
_encoder.Dispose();
_encoder = null;
_decoder.Dispose();
_decoder = null;
}
private void OnApplicationQuit()
{
StopEncoding();
}
Now here is the tcp send and receive, they are pretty much same for the client and the server
public void Send(byte[] data, int customParamLen = 0)
{
if (!socketReady)
{
return;
}
byte messageType = (1 << 3); // assume that 0000 1000 would be the Message type
byte[] message = data;
byte[] length = BitConverter.GetBytes(message.Length);
byte[] customParam = BitConverter.GetBytes(customParamLen); //length also 4/sizeof(int)
byte[] buffer = new byte[sizeof(int) + message.Length + 1 + customParam.Length];
buffer[0] = messageType;
//Enter length in the buffer
for (int i = 0; i < sizeof(int); i++)
{
buffer[i + 1] = length[i];
}
//Enter data in the buffer
for (int i = 0; i < message.Length; i++)
{
buffer[i + 1 + sizeof(int)] = message[i];
}
//Enter custom Param in the buffer
for (int i = 0; i < sizeof(int); i++)
{
buffer[i + 1 + sizeof(int) + message.Length] = customParam[i];
}
heavyStream.Write(buffer, 0, buffer.Length);
print("Writtin bytes");
}
if (heavyStream.DataAvailable)
{
print("Data Receiving YAY!");
//Get message Type
byte messageType = (byte)heavyStream.ReadByte();
//Get length of the Data
byte[] lengthBuffer = new byte[sizeof(int)];
int recv = heavyStream.Read(lengthBuffer, 0, lengthBuffer.Length);
if (recv == sizeof(int))
{
int messageLen = BitConverter.ToInt32(lengthBuffer, 0);
//Get the Data
byte[] messageBuffer = new byte[messageLen];
recv = heavyStream.Read(messageBuffer, 0, messageBuffer.Length);
if (recv == messageLen)
{
// messageBuffer contains the whole message ...
//Get length paramater needed for opus to decode
byte[] customParamAudioLen = new byte[sizeof(int)];
recv = heavyStream.Read(customParamAudioLen, 0, customParamAudioLen.Length);
if (recv == sizeof(int))
{
AudioReceivers(messageBuffer, BitConverter.ToInt32(customParamAudioLen, 0) - 5);
print("Done! Everything went straight as planned");
}
}
}
Now the problem is that the audio is choppy and has gaps in them, as the time flies the more out of sync it becomes.
UPDATE
Still not fixed.

It looks like you're just sending audio straight out with no jitter buffer on the receiving end. This means if you have any variability in latency you'll start to hear gaps.
What you need to do is buffer audio on the client side - until you have a good amount, say 400ms, then start playing. That gives you a buffer of extra time to account for jitter.
This is a very naive approach, but gives you something to play with - you'll probably want to look at adaptive jitter buffers, and probably switch to UDP instead of TCP to get better performance. With UDP you will need to deal with lost packets, out of order etc.
Have a look at Speex which has a Jitter Buffer https://github.com/xiph/speex or Mumble which uses Speex for VOIP https://github.com/mumble-voip/mumble

Marshal.Copy() causes System.StackOverflowException

I encountered a System.StackOverflowException problem when I was trying to Marshal.Copy()
Here is the screen shot of the code where exception happens:
Here is the function:
private static void ImageUpdate(IntPtr returnArray, ref int channels)
{
if (_prevTickCount == 0)
{
_sumTickCount = 0;
}
else
{
_sumTickCount = _sumTickCount * .75 + (Environment.TickCount - _prevTickCount) * .25;
}
//only copy to the buffer when pixel data is not being read
if (_pixelDataReady == false)
{
int width = 0;
int height = 0;
if (false == GetImageDimensions(ref width, ref height))
{
return;
}
_dataLength = width * height;
_colorChannels = channels;
if ((_pixelData == null) || (_pixelData.Length != (_dataLength * _colorChannels)))
{
_pixelData = new short[_dataLength * _colorChannels];
//_pixelDataHistogram = new int[_colorChannels][];
_pixelDataHistogram = new int[MAX_CHANNELS][];
if (_colorChannels == 1)
{
_pixelDataByte = new byte[_dataLength];
}
else
{
_pixelDataByte = new byte[_dataLength * 3];
}
//for (int i = 0; i < _colorChannels; i++)
for (int i = 0; i < MAX_CHANNELS; i++)
{
_pixelDataHistogram[i] = new int[PIXEL_DATA_HISTOGRAM_SIZE];
}
}
//2^n == FULL_RANGE_NORMALIZATION_FACTOR
const int SHIFT_VALUE = 6;
switch (_colorChannels)
{
case 1:
{
Marshal.Copy(returnArray, _pixelData, 0, _dataLength * _colorChannels);
//factor is derived by taking CAMERA_MAX_INTENSITY_VALUE/256
//const double FULL_RANGE_NORMALIZATION_FACTOR = 64.0;
//clear the histogram
for (int i = 0; i < PIXEL_DATA_HISTOGRAM_SIZE; i++)
{
_pixelDataHistogram[0][i] = 0;
}
for (int i = 0; i < _dataLength * _colorChannels; i++)
{
double valHist;
if (_pixelData[i] < 0)
{
valHist = (_pixelData[i] + 32768) >> SHIFT_VALUE;
}
else
{
valHist = (_pixelData[i]) >> SHIFT_VALUE;
}
_pixelDataHistogram[0][(byte)valHist]++;
}
}
break;
default:
{
Marshal.Copy(returnArray, _pixelData, 0, _dataLength * _colorChannels);
}
break;
}
_dataWidth = width;
_dataHeight = height;
_pixelDataReady = true;
ThorLog.Instance.TraceEvent(TraceEventType.Verbose, 1, "ImageUpdate pixeldata updated");
}
else
{
ThorLog.Instance.TraceEvent(TraceEventType.Verbose, 1, "ImageUpdate pixeldata not ready");
}
_prevTickCount = Environment.TickCount;
}
The whole idea is to copy image buffer from native code. This exception occurs only when image size is large 4K X 4K, but I dont have a problem processing a size below that.
I have no idea how I should correct this. Anyone care to educate? Thanks!

I traced it down, eventually, if was the returnArray which is not newed large enough to cause this problem.

Float to 16bit, Stereo. Improve?

So i am converting Float 32bit, to 16Bit in Stereo. And as i don´t fully understand it myself, it´s pretty much copy paste sadly.
But i wonder if it can be improved, in either quality or speed?
Not that any of them are terrible or anything.
void SendWaloop(object sender, NAudio.Wave.WaveInEventArgs e)
{
byte[] newArray16Bit = new byte[e.BytesRecorded / 2];
short two;
float value;
for (int i = 0, j = 0; i < e.BytesRecorded; i += 4, j += 2)
{
value = (BitConverter.ToSingle(e.Buffer, i));
two = (short)(value * short.MaxValue);
newArray16Bit[j] = (byte)(two & 0xFF);
newArray16Bit[j + 1] = (byte)((two >> 8) & 0xFF);
}
if (connect == true && MuteMic.Checked == false)
{
udpSend.Send(newArray16Bit, newArray16Bit.Length, otherPartyIP.Address.ToString(), 1500);
}
}
So well, it´s converting the buffer from 32bit to 16bit, and send´s it with UDP, nothing weird.
Though for me this looks very complex, but from what i understand, it´s just removing every 4 byte or something like that.
EDIT:
unsafe
{
byte[] newArray16Bit = new byte[e.BytesRecorded / 2];
fixed (byte* sourcePtr = e.Buffer)
fixed (byte* targetPtr = newArray16Bit)
{
float* sourceTyped = (float*)sourcePtr;
short* targetTyped = (short*)targetPtr;
int count = e.BytesRecorded / 4;
for (int i = 0; i < count; i++)
{
targetTyped[i] = (short)(sourceTyped[i] * short.MaxValue);
}
}
if (connect == true && MuteMic.Checked == false)
{
udpSend.Send(newArray16Bit, newArray16Bit.Length, otherPartyIP.Address.ToString(), 1500);
}
}
}

It would need testing, but I would probably try with some unsafe:
fixed(byte* sourcePtr = e.Buffer)
fixed(byte* targetPtr = newArray16Bit)
{
float* sourceTyped = (float*)sourcePtr;
short* targetTyped = (short*)targetPtr;
int count = e.BytesRecorded / 4;
for(int i = 0 ; i < count ; i++)
{
targetTyped[i] = (short)(sourceTyped[i] * short.MaxValue);
}
}
To show that working identically:
using System;
static class Program
{
static void Main()
{
byte[] raw1 = new byte[64 * 1024];
new Random(12345).NextBytes(raw1); // 64k of random data
var raw2 = (byte[])raw1.Clone(); // just to rule out corruption
var result1 = OriginalImplFromTopPost(raw1, raw1.Length - 20);
var result2 = MyImpl(raw2, raw2.Length - 20);
bool areSame = Convert.ToBase64String(result1) == Convert.ToBase64String(result2);
Console.WriteLine(areSame); // True
}
public static unsafe byte[] MyImpl(byte[] source, int byteCount)
{
byte[] newArray16Bit = new byte[byteCount / 2];
fixed (byte* sourcePtr = source)
fixed (byte* targetPtr = newArray16Bit)
{
float* sourceTyped = (float*)sourcePtr;
short* targetTyped = (short*)targetPtr;
int count = byteCount / 4;
for (int i = 0; i < count; i++)
{
targetTyped[i] = (short)(sourceTyped[i] * short.MaxValue);
}
}
return newArray16Bit;
}
public static byte[] OriginalImplFromTopPost(byte[] source, int byteCount)
{
byte[] newArray16Bit = new byte[byteCount / 2];
short two;
float value;
for (int i = 0, j = 0; i < byteCount; i += 4, j += 2)
{
value = (BitConverter.ToSingle(source, i));
two = (short)(value * short.MaxValue);
newArray16Bit[j] = (byte)(two & 0xFF);
newArray16Bit[j + 1] = (byte)((two >> 8) & 0xFF);
}
return newArray16Bit;
}
}