Unity Tcp /Ip Microphone Audio along Video Streaming - c#

I'm new to unity.
I'm trying to stream audio from the microphone with a live video from one app to the another, Currently, I have 2 apps. were app 1 is the server/sender and app 2 is the client/receiver. In-app 1 I successfully send the video bytes to the client. and on the client side, I'm also receiving all of the bytes. I'm using sockets and TCP.
Now the problem is I don't know how to send microphone's audio along with video from the server to the client.
Below code works perfectly for live video streaming from server side
using UnityEngine;
using System.Collections;
using System.IO;
using UnityEngine.UI;
using System;
using System.Text;
using System.Net;
using System.Net.Sockets;
using System.Threading;
using System.Collections.Generic;
public class Connecting : MonoBehaviour
{
WebCamTexture webCam;
public RawImage myImage;
public bool enableLog = false;
Texture2D currentTexture;
private TcpListener listner;
private const int port = 8010;
private bool stop = false;
private List<TcpClient> clients = new List<TcpClient>();
//This must be the-same with SEND_COUNT on the client
const int SEND_RECEIVE_COUNT = 15;
private void Start()
{
Application.runInBackground = true;
//Start WebCam coroutine
StartCoroutine(initAndWaitForWebCamTexture());
}
//Converts the data size to byte array and put result to the fullBytes array
void byteLengthToFrameByteArray(int byteLength, byte[] fullBytes)
{
//Clear old data
Array.Clear(fullBytes, 0, fullBytes.Length);
//Convert int to bytes
byte[] bytesToSendCount = BitConverter.GetBytes(byteLength);
//Copy result to fullBytes
bytesToSendCount.CopyTo(fullBytes, 0);
}
//Converts the byte array to the data size and returns the result
int frameByteArrayToByteLength(byte[] frameBytesLength)
{
int byteLength = BitConverter.ToInt32(frameBytesLength, 0);
return byteLength;
}
IEnumerator initAndWaitForWebCamTexture()
{
// Open the Camera on the desired device, in my case IPAD pro
webCam = new WebCamTexture();
// Get all devices , front and back camera
webCam.deviceName = WebCamTexture.devices[WebCamTexture.devices.Length - 1].name;
// request the lowest width and heigh possible
webCam.requestedHeight = 10;
webCam.requestedWidth = 10;
myImage.texture = webCam;
webCam.Play();
currentTexture = new Texture2D(webCam.width, webCam.height);
// Connect to the server
listner = new TcpListener(IPAddress.Any, port);
listner.Start();
while (webCam.width < 100)
{
yield return null;
}
//Start sending coroutine
StartCoroutine(senderCOR());
}
WaitForEndOfFrame endOfFrame = new WaitForEndOfFrame();
IEnumerator senderCOR()
{
bool isConnected = false;
TcpClient client = null;
NetworkStream stream = null;
// Wait for client to connect in another Thread
Loom.RunAsync(() =>
{
while (!stop)
{
// Wait for client connection
client = listner.AcceptTcpClient();
// We are connected
clients.Add(client);
isConnected = true;
stream = client.GetStream();
}
});
//Wait until client has connected
while (!isConnected)
{
yield return null;
}
LOG("Connected!");
bool readyToGetFrame = true;
byte[] frameBytesLength = new byte[SEND_RECEIVE_COUNT];
while (!stop)
{
//Wait for End of frame
yield return endOfFrame;
currentTexture.SetPixels(webCam.GetPixels());
byte[] pngBytes = currentTexture.EncodeToPNG();
//Fill total byte length to send. Result is stored in frameBytesLength
byteLengthToFrameByteArray(pngBytes.Length, frameBytesLength);
//Set readyToGetFrame false
readyToGetFrame = false;
Loom.RunAsync(() =>
{
//Send total byte count first
stream.Write(frameBytesLength, 0, frameBytesLength.Length);
LOG("Sent Image byte Length: " + frameBytesLength.Length);
//Send the image bytes
stream.Write(pngBytes, 0, pngBytes.Length);
LOG("Sending Image byte array data : " + pngBytes.Length);
//Sent. Set readyToGetFrame true
readyToGetFrame = true;
});
//Wait until we are ready to get new frame(Until we are done sending data)
while (!readyToGetFrame)
{
LOG("Waiting To get new frame");
yield return null;
}
}
}
void LOG(string messsage)
{
if (enableLog)
Debug.Log(messsage);
}
private void Update()
{
myImage.texture = webCam;
}
// stop everything
private void OnApplicationQuit()
{
if (webCam != null && webCam.isPlaying)
{
webCam.Stop();
stop = true;
}
if (listner != null)
{
listner.Stop();
}
foreach (TcpClient c in clients)
c.Close();
}
}
Below code works perfect for live video streaming from Client side
using UnityEngine;
using System.Collections;
using UnityEngine.UI;
using System.Net.Sockets;
using System.Net;
using System.IO;
using System;
public class reciver : MonoBehaviour
{
public RawImage image;
public bool enableLog = false;
const int port = 8010;
public string IP = "192.168.1.165";
TcpClient client;
Texture2D tex;
private bool stop = false;
//This must be the-same with SEND_COUNT on the server
const int SEND_RECEIVE_COUNT = 15;
// Use this for initialization
void Start()
{
Application.runInBackground = true;
tex = new Texture2D(0, 0);
client = new TcpClient();
//Connect to server from another Thread
Loom.RunAsync(() =>
{
LOGWARNING("Connecting to server...");
// if on desktop
client.Connect(IPAddress.Loopback, port);
// if using the IPAD
//client.Connect(IPAddress.Parse(IP), port);
LOGWARNING("Connected!");
imageReceiver();
});
}
void imageReceiver()
{
//While loop in another Thread is fine so we don't block main Unity Thread
Loom.RunAsync(() =>
{
while (!stop)
{
//Read Image Count
int imageSize = readImageByteSize(SEND_RECEIVE_COUNT);
LOGWARNING("Received Image byte Length: " + imageSize);
//Read Image Bytes and Display it
readFrameByteArray(imageSize);
}
});
}
//Converts the data size to byte array and put result to the fullBytes array
void byteLengthToFrameByteArray(int byteLength, byte[] fullBytes)
{
//Clear old data
Array.Clear(fullBytes, 0, fullBytes.Length);
//Convert int to bytes
byte[] bytesToSendCount = BitConverter.GetBytes(byteLength);
//Copy result to fullBytes
bytesToSendCount.CopyTo(fullBytes, 0);
}
//Converts the byte array to the data size and returns the result
int frameByteArrayToByteLength(byte[] frameBytesLength)
{
int byteLength = BitConverter.ToInt32(frameBytesLength, 0);
return byteLength;
}
/////////////////////////////////////////////////////Read Image SIZE from Server///////////////////////////////////////////////////
private int readImageByteSize(int size)
{
bool disconnected = false;
NetworkStream serverStream = client.GetStream();
byte[] imageBytesCount = new byte[size];
var total = 0;
do
{
var read = serverStream.Read(imageBytesCount, total, size - total);
//Debug.LogFormat("Client recieved {0} bytes", total);
if (read == 0)
{
disconnected = true;
break;
}
total += read;
} while (total != size);
int byteLength;
if (disconnected)
{
byteLength = -1;
}
else
{
byteLength = frameByteArrayToByteLength(imageBytesCount);
}
return byteLength;
}
/////////////////////////////////////////////////////Read Image Data Byte Array from Server///////////////////////////////////////////////////
private void readFrameByteArray(int size)
{
bool disconnected = false;
NetworkStream serverStream = client.GetStream();
byte[] imageBytes = new byte[size];
var total = 0;
do
{
var read = serverStream.Read(imageBytes, total, size - total);
//Debug.LogFormat("Client recieved {0} bytes", total);
if (read == 0)
{
disconnected = true;
break;
}
total += read;
} while (total != size);
bool readyToReadAgain = false;
//Display Image
if (!disconnected)
{
//Display Image on the main Thread
Loom.QueueOnMainThread(() =>
{
displayReceivedImage(imageBytes);
readyToReadAgain = true;
});
}
//Wait until old Image is displayed
while (!readyToReadAgain)
{
System.Threading.Thread.Sleep(1);
}
}
void displayReceivedImage(byte[] receivedImageBytes)
{
tex.LoadImage(receivedImageBytes);
image.texture = tex;
}
// Update is called once per frame
void Update()
{
}
void LOG(string messsage)
{
if (enableLog)
Debug.Log(messsage);
}
void LOGWARNING(string messsage)
{
if (enableLog)
Debug.LogWarning(messsage);
}
void OnApplicationQuit()
{
LOGWARNING("OnApplicationQuit");
stop = true;
if (client != null)
{
client.Close();
}
}
}
My Achievement for video so far is below, in below code audio is successfully playing from microphone
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
public class Audio1 : MonoBehaviour
{
const int FREQUENCY = 44100;
AudioClip mic;
int lastPos, pos;
// Use this for initialization
void Start()
{
mic = Microphone.Start(null, true, 10, FREQUENCY);
AudioSource audio = GetComponent<AudioSource>();
audio.clip = AudioClip.Create("test", 10 * FREQUENCY, mic.channels, FREQUENCY, false);
audio.loop = true;
}
// Update is called once per frame
void Update()
{
if ((pos = Microphone.GetPosition(null)) > 0)
{
if (lastPos > pos) lastPos = 0;
if (pos - lastPos > 0)
{
// Allocate the space for the sample.
float[] sample = new float[(pos - lastPos) * mic.channels];
// Get the data from microphone.
mic.GetData(sample, lastPos);
// Put the data in the audio source.
AudioSource audio = GetComponent<AudioSource>();
audio.clip.SetData(sample, lastPos);
if (!audio.isPlaying) audio.Play();
lastPos = pos;
}
}
}
void OnDestroy()
{
Microphone.End(null);
}
}

Related

C#+VideoLan.LibVLC Sample Audio Stream and save it to disk

I am using VideoLan.VLC to get 20 seconds of an audio stream every 30 seconds.
I have a loop, something like
private LibVLC libvlc = new LibVLC();
private MediaPlayer mediaPlayer = null;
private Media Media = null;
private string AudioSampleFileName { get {
return "audio_" + DateTime.Now.ToString("yyyyMMdd_HHmmss")+
".ts";
} }
public void Start(){
mediaPlayer = new MediaPlayer(libvlc);
while(...){
Get_20_seconds_audio_sample();
Wait_30_Seconds();
}
}
public void Get_sample(Uri playPathUri, string FileName)
{
var currentDirectory = Path.GetDirectoryName(Assembly.GetEntryAssembly().Location);
var destination = Path.Combine(currentDirectory, FileName);
var mediaOptions = new string[]
{
":sout=#file{dst=" + destination + ",channels=1,samplerate=16000}",
":sout-keep"
};
if (Media == null)
{
Media = new Media(libvlc, playPathUri, mediaOptions);
mediaPlayer.Media = Media;
}
else {
Media.AddOption(":sout=#file{dst=" + destination + ",channels=1,samplerate=16000}");
}
mediaPlayer.Play();
}
public void Get_20_seconds_audio_sample(){
Get_sample(RadioURI,AudioSampleFileName);
Wait_20_seconds();
Stop();
}
public void Stop()
{
mediaPlayer.Stop();
}
The Problem is that radio streming ususally starts with a commercial that lasts about 25 seconds. Every sample plays the commercial. It seems that Stop() closes the stream until Play() is called again and it restart the stream. I tired to pause the audio but, well...it makes no much sense to pause and play.
I can accept to get the commercial only in the first sample but then I want the regular radio audio. Is there a way not to close the stream every time? I am not tied to VideoLan dll so I can start from scratch if you have a better way to do it.
Use Audio callbacks and save the audio stream yourself. Full sample:
class Program
{
// This sample shows you how you can use SetAudioFormatCallback and SetAudioCallbacks. It does two things:
// 1) Play the sound from the specified video using NAudio
// 2) Extract the sound into a file using NAudio
static void Main(string[] args)
{
Core.Initialize();
using var libVLC = new LibVLC(enableDebugLogs: true);
using var media = new Media(libVLC,
new Uri("http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ElephantsDream.mp4"),
":no-video");
using var mediaPlayer = new MediaPlayer(media);
using var outputDevice = new WaveOutEvent();
var waveFormat = new WaveFormat(8000, 16, 1);
var writer = new WaveFileWriter("sound.wav", waveFormat);
var waveProvider = new BufferedWaveProvider(waveFormat);
outputDevice.Init(waveProvider);
mediaPlayer.SetAudioFormatCallback(AudioSetup, AudioCleanup);
mediaPlayer.SetAudioCallbacks(PlayAudio, PauseAudio, ResumeAudio, FlushAudio, DrainAudio);
mediaPlayer.Play();
mediaPlayer.Time = 20_000; // Seek the video 20 seconds
outputDevice.Play();
Console.WriteLine("Press 'q' to quit. Press any other key to pause/play.");
while (true)
{
if (Console.ReadKey().KeyChar == 'q')
break;
if (mediaPlayer.IsPlaying)
mediaPlayer.Pause();
else
mediaPlayer.Play();
}
void PlayAudio(IntPtr data, IntPtr samples, uint count, long pts)
{
int bytes = (int)count * 2; // (16 bit, 1 channel)
var buffer = new byte[bytes];
Marshal.Copy(samples, buffer, 0, bytes);
waveProvider.AddSamples(buffer, 0, bytes);
writer.Write(buffer, 0, bytes);
}
int AudioSetup(ref IntPtr opaque, ref IntPtr format, ref uint rate, ref uint channels)
{
channels = (uint)waveFormat.Channels;
rate = (uint)waveFormat.SampleRate;
return 0;
}
void DrainAudio(IntPtr data)
{
writer.Flush();
}
void FlushAudio(IntPtr data, long pts)
{
writer.Flush();
waveProvider.ClearBuffer();
}
void ResumeAudio(IntPtr data, long pts)
{
outputDevice.Play();
}
void PauseAudio(IntPtr data, long pts)
{
outputDevice.Pause();
}
void AudioCleanup(IntPtr opaque) { }
}
}

Sending Pictures over Network with Unity C#

I am currently working with the HoloLens and thus Unity. The idea is to make a webcam Livestream with as little delay as possible to an instance of FFmpeg on a different device for encoding. However, I run into some issues.
My first problem is the Conversion from a Color32 Array to a Byte Array. It creates a huge amount of bandwidth, especially when I am trying to record and stream with 30fps and a 1280x720 resolution.
Here is some code:
IEnumerator Init()
{
webCam = new WebCamTexture(1280, 720, 30);
webCam.Play();
image.texture = webCam;
Debug.Log("WebCam Width: " + webCam.width);
Debug.Log("WebCam Height: " + webCam.height);
currentTexture = new Texture2D(webCam.width, webCam.height);
data = new Color32[webCam.width * webCam.height];
listener = new TcpListener(IPAddress.Any, 10305);
listener.Start();
while (webCam.width < 100)
{
yield return null;
}
StartCoroutine("Recording");
}
WaitForEndOfFrame endOfFrame = new WaitForEndOfFrame();
IEnumerator Recording()
{
TcpClient tcpClient = null;
NetworkStream tcpStream = null;
bool isConnected = false;
Loom.RunAsync(() =>
{
while(!stopCapture)
{
tcpClient = listener.AcceptTcpClient();
Debug.Log("Client Connected!");
isConnected = true;
tcpStream = tcpClient.GetStream();
}
});
while(!isConnected)
{
yield return endOfFrame;
}
readyToGetFrame = true;
byte[] messageLength = new byte[SEND_RECEIVE_COUNT];
while(!stopCapture)
{
yield return endOfFrame;
webCam.GetPixels32(data);
byte[] webCamBytes = Utilities.Color32ArrayToByteArray(data);
readyToGetFrame = false;
Loom.RunAsync(() => {
tcpStream.Write(webCamBytes, 0, webCamBytes.Length);
readyToGetFrame = true;
});
while (!readyToGetFrame)
{
yield return endOfFrame;
}
}
}
public static class Utilities
{
public static byte[] Color32ArrayToByteArray(Color32[] colors)
{
if (colors == null || colors.Length == 0)
{
return null;
}
int lengthOfColor32 = Marshal.SizeOf(typeof(Color32));
int byteArrayLength = lengthOfColor32 * colors.Length;
byte[] bytes = new byte[byteArrayLength];
GCHandle handle = default(GCHandle);
try
{
handle = GCHandle.Alloc(colors, GCHandleType.Pinned);
IntPtr ptr = handle.AddrOfPinnedObject();
Marshal.Copy(ptr, bytes, 0, byteArrayLength);
}
finally
{
if(handle != default(GCHandle))
{
handle.Free();
}
}
return bytes;
}
}
To fix this problem, I tried to use EncodePNG and EncodeJPG instead. But there is my second problem. If I use these methods, I got a big loss of performance.
So instead of this
webCam.GetPixels32(data);
byte[] webCamBytes = Utilities.Color32ArrayToByteArray(data);
I use this
currentTexture.SetPixels(webCam.GetPixels());
byte[] webCamBytes = currentTexture.EncodeToJPG();
Thus my question:
Is there any way to reduce the bandwith by maybe cutting off the alpha values as they are not needed? Or is there another good way such as converting to a different color or picture format?
Because right now I ran out of usable ideas and feel kinda stuck. Might just be me though ;-)
PS: Using external libraries is a tad difficult as it needs to run on HoloLens and due to other regulations.
Thank you in advance!

MediaFoundationResampler of audio loopback stream is alway read as 0 bytes

I can see that the system audio for WasapiLoopbackCapture is not the right rate to try and get Microsoft Windows' audio to text recognition working (only 8/16 is supported, eg AudioBitsPerSample.Sixteen ), from the output of this program. The odd thing I cannot figure out is why the resampler in this code is never called. I set a breakpoint - even wait 5s for the system audio to play and be placed in the buffer. I am trying to get system sound, to text using the audio speech recognizer. I can see that bytes are written to captureStream during debugging, so why does the resampler never output bytes? - The Console.WriteLine("Never getting here"); is never getting there.
using System;
using System.Speech.Recognition;
using NAudio.Wave;
using NAudio.CoreAudioApi.Interfaces;
using NAudio.CoreAudioApi;
using System.IO;
using System.Speech.AudioFormat;
namespace SpeechRecognitionApp
{
class FakeStreamer : Stream
{
public bool bExit = false;
Stream stream;
Stream client;
public FakeStreamer(Stream client)
{
this.client = client;
this.stream = client;
}
public override bool CanRead
{
get { return stream.CanRead; }
}
public override bool CanSeek
{
get { return false; }
}
public override bool CanWrite
{
get { return stream.CanWrite; }
}
public override long Length
{
get { return -1L; }
}
public override long Position
{
get { return 0L; }
set { }
}
public override long Seek(long offset, SeekOrigin origin)
{
return 0L;
}
public override void SetLength(long value)
{
stream.SetLength(value);
}
public override int Read(byte[] buffer, int offset, int count)
{
int len = 0, c = count;
while (c > 0 && !bExit)
{
try
{
len = stream.Read(buffer, offset, c);
}
catch (Exception e)
{
Console.WriteLine("ouch");
}
/*if (!client.Connected || len == 0)
{
//Exit read loop
return 0;
}*/
offset += len;
c -= len;
}
return count;
}
public override void Write(byte[] buffer, int offset, int count)
{
stream.Write(buffer, offset, count);
}
public override void Close()
{
stream.Close();
base.Close();
}
public override void Flush()
{
stream.Flush();
}
}
class Program
{
static void Main(string[] args)
{
// Create an in-process speech recognizer for the en-US locale.
using (
SpeechRecognitionEngine recognizer =
new SpeechRecognitionEngine(
new System.Globalization.CultureInfo("en-US")))
{
// Create and load a dictation grammar.
recognizer.LoadGrammar(new DictationGrammar());
// Add a handler for the speech recognized event.
recognizer.SpeechRecognized +=
new EventHandler<SpeechRecognizedEventArgs>(recognizer_SpeechRecognized);
// Configure input to the speech recognizer.
//recognizer.SetInputToDefaultAudioDevice();
WasapiLoopbackCapture capture = new WasapiLoopbackCapture();
Stream captureStream = new System.IO.MemoryStream();
//Stream buffStream = new FakeStreamer(captureStream);
capture.DataAvailable += (s, a) =>
{
//It is getting here.
captureStream.Write(a.Buffer, 0, a.BytesRecorded);
};
Console.WriteLine(capture.WaveFormat.AverageBytesPerSecond);
Console.WriteLine(capture.WaveFormat.BitsPerSample);
var newFormat = new WaveFormat(8000, 16, 1);
//using (var conversionStream = new WaveFormatConversionStream(newFormat, capture)
//capture.StartRecording();
using (var resampler = new MediaFoundationResampler(new NAudio.Wave.RawSourceWaveStream(captureStream, capture.WaveFormat), newFormat))
{
Stream captureConvertStream = new System.IO.MemoryStream();
resampler.ResamplerQuality = 60;
//WaveFileWriter.WriteWavFileToStream(captureConvertStream, resampler);
//recognizer.SetInputToDefaultAudioDevice();
Stream buffStream = new FakeStreamer(captureConvertStream);
recognizer.SetInputToAudioStream(buffStream, new SpeechAudioFormatInfo(
8000, AudioBitsPerSample.Sixteen, AudioChannel.Mono));
// Start asynchronous, continuous speech recognition.
recognizer.RecognizeAsync(RecognizeMode.Multiple);
capture.StartRecording();
//Never getting to the resampler, the read is always zero!? even if waiting 5s for the audio to buffer.
System.Threading.Thread.Sleep(5000);
var arr = new byte[128];
while (resampler.Read(arr, 0, arr.Length) > 0)
{
captureConvertStream.Write(arr, 0, arr.Length);
Console.WriteLine("Never getting here");
}
// Keep the console window open.
while (true)
{
Console.ReadLine();
}
}
}
}
// Handle the SpeechRecognized event.
static void recognizer_SpeechRecognized(object sender, SpeechRecognizedEventArgs e)
{
Console.WriteLine("Recognized text: " + e.Result.Text);
}
}
}

WasapiLoopbackCapture internal audio recognition gives jibberish and text when no audio

I finally have built a program to listen to the internal audio loopback using NAudio, and output recognized text. The problem is it listens, and always says, eg:
Recognized text: had
Recognized text: had
Recognized text: had
Recognized text: had
Recognized text: had had phone Le K add phone Laton
Recognized text: had phone looked had phone looked had phone looked had phone lo
oked zone
Recognized text: had phone lines to had, had phone looked had phone looked had p
hone line had phone
Recognized text: had phone line had phone looked had phone
Recognized text: had phone looked had phone looked had phone line had phone
Recognized text: had phone looked had phone look to had pot they had phone lit o
nly had phone
Recognized text: had phone line had phone looked had phone line to had to had ph
one
Recognized text: had phone line had phone looked had phone looked had phone
Recognized text: had phone line had phone looked had phone looked had phone line
10 only T had phone
Recognized text: had phone line had
Recognized text: had phone line had phone looked had phone line had
Recognized text: had phone Le tone looked had
Recognized text: had phone looked had phone looked had phone
Recognized text: had phone line had phone line had phone licked had phone
Recognized text: had phone lines to had popped the own
and similar nonsense, but even when I pause audio it just shows "Recognized text: had" or "an" again and again and again. When I unpause audio it keeps unsuccessfully recognizing the internal audio. Is there a way I can fix this, or at least get a wav of what it's trying to send to the Microsoft speech recognition recognizer?
using System;
using System.Speech.Recognition;
using NAudio.Wave;
using NAudio.CoreAudioApi.Interfaces;
using NAudio.CoreAudioApi;
using System.IO;
using System.Speech.AudioFormat;
using NAudio.Wave.SampleProviders;
using NAudio.Utils;
using System.Threading;
using System.Collections.Generic;
namespace SpeechRecognitionApp
{
class SpeechStreamer : Stream
{
private AutoResetEvent _writeEvent;
private List<byte> _buffer;
private int _buffersize;
private int _readposition;
private int _writeposition;
private bool _reset;
public SpeechStreamer(int bufferSize)
{
_writeEvent = new AutoResetEvent(false);
_buffersize = bufferSize;
_buffer = new List<byte>(_buffersize);
for (int i = 0; i < _buffersize; i++)
_buffer.Add(new byte());
_readposition = 0;
_writeposition = 0;
}
public override bool CanRead
{
get { return true; }
}
public override bool CanSeek
{
get { return false; }
}
public override bool CanWrite
{
get { return true; }
}
public override long Length
{
get { return -1L; }
}
public override long Position
{
get { return 0L; }
set { }
}
public override long Seek(long offset, SeekOrigin origin)
{
return 0L;
}
public override void SetLength(long value)
{
}
public override int Read(byte[] buffer, int offset, int count)
{
int i = 0;
while (i < count && _writeEvent != null)
{
if (!_reset && _readposition >= _writeposition)
{
_writeEvent.WaitOne(100, true);
continue;
}
buffer[i] = _buffer[_readposition + offset];
_readposition++;
if (_readposition == _buffersize)
{
_readposition = 0;
_reset = false;
}
i++;
}
return count;
}
public override void Write(byte[] buffer, int offset, int count)
{
for (int i = offset; i < offset + count; i++)
{
_buffer[_writeposition] = buffer[i];
_writeposition++;
if (_writeposition == _buffersize)
{
_writeposition = 0;
_reset = true;
}
}
_writeEvent.Set();
}
public override void Close()
{
_writeEvent.Close();
_writeEvent = null;
base.Close();
}
public override void Flush()
{
}
}
class FakeStreamer : Stream
{
public bool bExit = false;
Stream stream;
Stream client;
public FakeStreamer(Stream client)
{
this.client = client;
this.stream = client;
}
public override bool CanRead
{
get { return stream.CanRead; }
}
public override bool CanSeek
{
get { return false; }
}
public override bool CanWrite
{
get { return stream.CanWrite; }
}
public override long Length
{
get { return -1L; }
}
public override long Position
{
get { return 0L; }
set { }
}
public override long Seek(long offset, SeekOrigin origin)
{
return 0L;
}
public override void SetLength(long value)
{
stream.SetLength(value);
}
public override int Read(byte[] buffer, int offset, int count)
{
int len = 0, c = count;
while (c > 0 && !bExit)
{
//try {
len = stream.Read(buffer, offset, c);
/*}
catch (Exception e)
{
Console.WriteLine("ouch");
}
if (!client.Connected || len == 0)
{
//Exit read loop
return 0;
}*/
offset += len;
c -= len;
}
return count;
}
public override void Write(byte[] buffer, int offset, int count)
{
stream.Write(buffer, offset, count);
}
public override void Close()
{
stream.Close();
base.Close();
}
public override void Flush()
{
stream.Flush();
}
}
class Program
{
static void Main(string[] args)
{
// Create an in-process speech recognizer for the en-US locale.
using (
SpeechRecognitionEngine recognizer =
new SpeechRecognitionEngine(
new System.Globalization.CultureInfo("en-US")))
{
// Create and load a dictation grammar.
recognizer.LoadGrammar(new DictationGrammar());
// Add a handler for the speech recognized event.
recognizer.SpeechRecognized +=
new EventHandler<SpeechRecognizedEventArgs>(recognizer_SpeechRecognized);
// Configure input to the speech recognizer.
//recognizer.SetInputToDefaultAudioDevice();
WasapiLoopbackCapture capture = new WasapiLoopbackCapture();
BufferedWaveProvider WaveBuffer = new BufferedWaveProvider(capture.WaveFormat);
WaveBuffer.DiscardOnBufferOverflow = true;
//WaveBuffer.ReadFully = false;
WaveToSampleProvider sampleStream = new WaveToSampleProvider(WaveBuffer);
StereoToMonoSampleProvider monoStream = new StereoToMonoSampleProvider(sampleStream)
{
LeftVolume = 1f,
RightVolume = 1f
};
//Downsample to 8000 https://stackoverflow.com/questions/48233099/capture-audio-from-wasapiloopbackcapture-and-convert-to-mulaw
WdlResamplingSampleProvider resamplingProvider = new WdlResamplingSampleProvider(monoStream, 16000);
SampleToWaveProvider16 ieeeToPcm = new SampleToWaveProvider16(resamplingProvider);
var arr = new byte[128];
Stream captureConvertStream = new System.IO.MemoryStream();
capture.StartRecording();
//outputStream = new MuLawConversionProvider(ieeeToPcm);
Stream captureStream = new System.IO.MemoryStream();
//Stream buffStream = new FakeStreamer(captureStream);
capture.DataAvailable += (s, a) =>
{
//It is getting here.
//captureStream.Write(a.Buffer, 0, a.BytesRecorded);
WaveBuffer.AddSamples(a.Buffer, 0, a.BytesRecorded);
};
Console.WriteLine(capture.WaveFormat.AverageBytesPerSecond);
Console.WriteLine(capture.WaveFormat.BitsPerSample);
//var newFormat = new WaveFormat(8000, 16, 1);
//using (var conversionStream = new WaveFormatConversionStream(newFormat, capture)
//capture.StartRecording();
//using (var resampler = new MediaFoundationResampler(new NAudio.Wave.RawSourceWaveStream(captureStream, capture.WaveFormat), newFormat))
//{
//resampler.ResamplerQuality = 60;
//WaveFileWriter.WriteWavFileToStream(captureConvertStream, resampler);
//recognizer.SetInputToDefaultAudioDevice();
//Stream buffStream = new FakeStreamer(captureConvertStream);
Stream buffStream = new SpeechStreamer(2048);
//recognizer.SetInputToWaveStream(buffStream);
recognizer.SetInputToAudioStream(buffStream, new SpeechAudioFormatInfo(
16000, AudioBitsPerSample.Eight, AudioChannel.Mono));
// Start asynchronous, continuous speech recognition.
recognizer.RecognizeAsync(RecognizeMode.Multiple);
/*System.Threading.Thread.Sleep(5000);
works when playing anything
var floata = new float[128];
while(monoStream.Read(floata, 0, floata.Length) > 0 )
{
Console.WriteLine(arr.Length);
}*/
while (ieeeToPcm.Read(arr, 0, arr.Length) > 0)
{
//Console.Write("Writing PCM ");
//Console.WriteLine(arr.Length);
//captureConvertStream.Write(arr, 0, arr.Length);
buffStream.Write(arr, 0, arr.Length);
}
Console.WriteLine("end");
/*capture.StartRecording();
//Never getting to the resampler, the read is always zero!? even if waiting 5s for the audio to buffer.
System.Threading.Thread.Sleep(5000);
var arr = new byte[128];
while (resampler.Read(arr, 0, arr.Length) > 0)
{
captureConvertStream.Write(arr, 0, arr.Length);
Console.WriteLine("Never getting here");
}
// Keep the console window open.
while (true)
{
Console.ReadLine();
}*/
//}
}
}
// Handle the SpeechRecognized event.
static void recognizer_SpeechRecognized(object sender, SpeechRecognizedEventArgs e)
{
Console.WriteLine("Recognized text: " + e.Result.Text);
}
}
}
That SpeechStreamer class has some problems, I cannot really see what its purpose is. I tried. Also looking at wavefile dumps from your implementation, the audio is really choppy, with long pauses between the samples. This might be what is throwing the speech recognizer off. This is an example: Windows Volume Adjutment Sound From Your Code
As you may hear, it is pretty choppy with a lot of silence between. The Voice Recognition part recognizes this as : "ta ta ta ta ta ta..."
I had to rewrite your code a bit to dump a wave file, since the Read method of your SpeechStream causes an eternal loop when used to read its contents.
To dump a wave file you could do the following:
var buffer = new byte[2048];
using (var writer = new WaveFileWriter("tmp.wav", ieeeToPcm.WaveFormat))
{
//buffStream is changed to a MemoryStream for this to work.
buffStream.Seek(0,SeekOrigin.Begin);
while (buffStream.Read(buffer, 0, buffer.Length)>0)
{
writer.Write(buffer, 0, buffer.Length);
}
}
Or you can do it when you read from your SampleToWaveProvider16:
var writer = new WaveFileWriter("dump.wav", ieeeToPcm.WaveFormat);
while (ieeeToPcm.Read(arr, 0, arr.Length) > 0)
{
if (Console.KeyAvailable && Console.ReadKey().Key == ConsoleKey.Escape)
break;
buffStream.Write(arr, 0, arr.Length);
writer.Write(arr, 0, arr.Length);
}
I just added the ability to hit Escape to exit the loop.
Now I do wonder why you are using NAudio? Why not use the methods native to the Sound.Speech API?
class Program
{
private static ManualResetEvent _done;
static void Main(string[] args)
{
_done = new ManualResetEvent(false);
using (SpeechRecognitionEngine recognizer = new SpeechRecognitionEngine(new CultureInfo("en-US")))
{
recognizer.LoadGrammar(new DictationGrammar());
recognizer.SpeechRecognized += RecognizedSpeech;
recognizer.SetInputToDefaultAudioDevice();
recognizer.RecognizeAsync(RecognizeMode.Multiple);
_done.WaitOne();
}
}
private static void RecognizedSpeech(object sender, SpeechRecognizedEventArgs e)
{
if (e.Result.Text.Contains("exit"))
{
_done.Set();
}
Console.WriteLine(e.Result.Text);
}
}

Continuously getting data from heart rate monitor

I've been working on a project that reads a person's heart rate from a Polar H7 hrm. I've been successful in connecting the device and getting the heart rate which the program shows as text in the UI. However, there are instances where the program suddenly stops getting input from the device.
I have already checked the connection of the device to my Win 10 laptop and saw that it was stable, there were also no exceptions getting thrown by the program. The text simply stops changing.
Here is the code I've written:
public sealed partial class MainPage : Page
{
private GattDeviceService device;
public MainPage()
{
this.InitializeComponent();
init();
}
async void init()
{
var devices = await DeviceInformation.FindAllAsync(GattDeviceService.GetDeviceSelectorFromUuid(GattServiceUuids.HeartRate));
Status.Text = devices.Count.ToString();
device = await GattDeviceService.FromIdAsync(devices[0].Id);
enableSensor();
}
private async void enableSensor()
{
IReadOnlyList<GattCharacteristic> characteristicList;
characteristicList = device.GetAllCharacteristics();
if (characteristicList != null)
{
GattCharacteristic characteristic = characteristicList[0];
if (characteristic.CharacteristicProperties.HasFlag(GattCharacteristicProperties.Notify))
{
characteristic.ValueChanged += SendNotif;
await characteristic.WriteClientCharacteristicConfigurationDescriptorAsync(GattClientCharacteristicConfigurationDescriptorValue.Notify);
}
}
}
async void SendNotif(GattCharacteristic sender, GattValueChangedEventArgs eventArgs)
{
if (eventArgs.CharacteristicValue.Length != 0) {
byte[] hrData = new byte[eventArgs.CharacteristicValue.Length];
DataReader.FromBuffer(eventArgs.CharacteristicValue).ReadBytes(hrData);
var hrValue = ProcessData(hrData);
await Dispatcher.RunAsync(Windows.UI.Core.CoreDispatcherPriority.Normal, () =>
{
Status.Text = hrValue.ToString();
});
}
}
private int ProcessData(byte[] data)
{
// Heart Rate profile defined flag values
const byte heartRateValueFormat = 0x01;
byte currentOffset = 0;
byte flags = data[currentOffset];
bool isHeartRateValueSizeLong = ((flags & heartRateValueFormat) != 0);
currentOffset++;
ushort heartRateMeasurementValue;
if (isHeartRateValueSizeLong)
{
heartRateMeasurementValue = (ushort)((data[currentOffset + 1] << 8) + data[currentOffset]);
currentOffset += 2;
}
else
{
heartRateMeasurementValue = data[currentOffset];
}
return heartRateMeasurementValue;
}
}

Categories