We Keep Coding

How to send live audio stream to IBM Watson Speech-To-Text immediately without saving audio file locally using C# and IBM Watson SDK?

I have a requirement to use IBM Watson SDK to record the audio using microphone and send it to IBM Watson speech-to-text using C#. I am able to achieve this functionality by saving the audio file locally and then sending it using NAudio library. But my requirement is to use streaming mode to send live audio to IBM Watson Speech-to-Text service without storing the audio file physically.
I am not able to find RecognizeUsingWebSocket service in the SDK. I am able to find only Recognize service.
Below is my code which is used to save audio file locally.
Can anyone please help me to achieve it ?
using System;
using System.IO;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using IBM.Cloud.SDK.Core.Authentication.Iam;
using IBM.Watson.Assistant.v1.Model;
using IBM.Watson.Assistant.v1;
using IBM.Cloud.SDK.Core.Authentication.Bearer;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NAudio.Wave;
using System.Net.WebSockets;
using IBM.Cloud.SDK.Core.Http;
using IBM.Watson.SpeechToText.v1;
namespace watsonConsole
{
class Program
{
string apikey = "";
string sttApiKey = "";
string url = "";
string stturl = "";
string versionDate = "";
string workspaceId = "";
static public AssistantService service;
static public SpeechToTextService sttservice;
private WaveInEvent waveIn;
private WaveFormat format = new WaveFormat(16000, 16, 1);
private ClientWebSocket ws;
static public WaveFileWriter waveFile;
static void Main(string[] args)
{
Program pr = new Program();
BearerTokenAuthenticator authenticator = new BearerTokenAuthenticator(
bearerToken: pr.apikey);
service = new AssistantService(pr.versionDate, authenticator);
service.SetServiceUrl(pr.url);
BearerTokenAuthenticator sttauthenticator = new BearerTokenAuthenticator(
bearerToken: pr.sttApiKey);
sttservice = new SpeechToTextService(sttauthenticator);
sttservice.SetServiceUrl(pr.stturl);
WaveInCapabilities deviceInfo = WaveIn.GetCapabilities(0);
Console.WriteLine("Now recording...");
WaveInEvent waveSource = new WaveInEvent();
waveSource.DeviceNumber = 0;
waveSource.WaveFormat = new WaveFormat(44100, 1);
waveSource.DataAvailable += new EventHandler<WaveInEventArgs>(waveSource_DataAvailable);
string tempFile = (#"C:/watsonConsole/bin/Debug/test/testaudiotest.wav");
waveFile = new WaveFileWriter(tempFile, waveSource.WaveFormat);
waveSource.StartRecording();
Console.WriteLine("Press enter to stop");
Console.ReadLine();
waveSource.StopRecording();
waveFile.Dispose();
pr.Recognize();
Console.WriteLine("done");
Console.ReadKey();
}
static void waveSource_DataAvailable(object sender, WaveInEventArgs e)
{
waveFile.Write(e.Buffer, 0, e.BytesRecorded);
}
public void StartRecording()
{
waveIn = new WaveInEvent
{
BufferMilliseconds = 50,
DeviceNumber = 0,
WaveFormat = format
};
waveIn.StartRecording();
}
public void Recognize()
{
var result = sttservice.Recognize(
audio: File.ReadAllBytes("test/testaudiotest.wav"),
contentType: "audio/wav",
wordAlternativesThreshold: 0.9f,
languageCustomizationId: "",
acousticCustomizationId: "",
customizationWeight: 0.7,
smartFormatting: true
);
Console.WriteLine(result.Response);
}
}
}

Unity3d client - Python server real-time communication

I am trying to create a client server app using Unity3D and python. I am trying to send big amount of data from server to client in real time. In order to do that i divided the data to be received in smaller bulks so the serialization/deserialization to be faster so the update of unity objects based on data to be faster as well.
I approached the problem like this:
I open 5 sockets and each try to handle the request based on an identifier
On client I use parallel.foreach for sending the five identifiers and receiving then in parallel.
The issue is that client.Receive blocks when there is no data to receive. I tried to set a timeout but it is not really helpful.
Also the length of data is variable for the packages received.
Python server implementation is similar to this but it is not that relevant i assume tho: Is it possible to run multiple asyncio in the same time in python?
Socket class:
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using System.Net.Sockets;
using System;
using System.IO;
public class SocketConnector
{
public string ip = "127.0.0.1";
public int port = 60000;
private Socket client;
[SerializeField]
private int dataOut;
private byte[] dataIn; //debugging
public SocketConnector() { }
public void establishConnection(int port)
{
client = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
client.Connect(ip, port);
}
public byte[] ServerRequest(int dataOut)
{
//print("request");
this.dataOut = dataOut; //debugging
this.dataIn = SendAndReceive(dataOut); //debugging
return this.dataIn;
}
private byte[] SendAndReceive(int dataOut)
{
byte[] z = new byte[1024 * 1024 * 1500];
try
{
byte[] intBytes = BitConverter.GetBytes(dataOut);
client.Send(intBytes);
client.ReceiveTimeout = 2000;
//allocate and receive bytes
byte[] bytes = new byte[1024 * 1024 * 1500];
Array.Clear(bytes, 0, bytes.Length);
int lenght = 1, byteIndex = 0;
lenght = client.Receive(bytes); // is blocking here cos no data is received sometime
Buffer.BlockCopy(bytes, 0, z, byteIndex, lenght);
byteIndex += lenght;
Array.Resize(ref z, byteIndex);
Debug.Log("request received");
}
catch (Exception e)
{
Debug.Log("smth happened");
}
client.Close();
return z;
}
}
Main class:
...
void Start()
{
ThreadCompoennt threadComponent = new ThreadComponent(socketConnector, this, featureMapCreator);
threadComponent.init();
}
...
Thread component:
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using System.Threading;
using System.Linq;
using System.Threading.Tasks;
using MessagePack;
using System;
public class ThreadComponent
{
Thread thread;
SocketConnector socketConnector;
Main main;
Dictionary<string, Dictionary<string, ushort[][]>> feature;
StructHelper structHelper;
byte[] z;
public ThreadComponent(SocketConnector socketConnector, Main main, FeatureMapCreator featureMapCreator)
{
this.featureMapCreator = featureMapCreator;
this.socketConnector = socketConnector;
this.main = main;
}
public void init()
{
thread = new System.Threading.Thread(doLogic);
thread.IsBackground = true;
thread.Start();
}
void doLogic()
{
List<int> integerList = Enumerable.Range(1, 5).ToList();
while (true)
{
Thread.Sleep(2000);
Parallel.ForEach(integerList, i =>
{
socketConnector.establishConnection(60000 + i);
doLogicGetData(i, socketConnector);
});
}
}
void doLogicGetData(int batchIdentifier, SocketConnector socketConnector)
{
z = socketConnector.ServerRequest(batchIdentifier);
if (z.Length != 0)
{
this.feature = MessagePackSerializer.Deserialize<Dictionary<string, Dictionary<string,ushort[][]>>>(z);
..... do update based on batch identifier
}
}

NetworkStream cuts off first 4 bytes when reading

I ran into a strange problem. When I'm trying to send the file via a TCP socket, the first 4 bytes of sended information cuts off.
That is sending and receives pieces of code.
Client side
for (var i = 0; i < fileContentByte.Length; i += buffer.Length)
{
var size = (i + buffer.Length > fileContentByte.Length) ? fileContentByte.Length - i : buffer.Length;
clientSocket.Write(fileContentByte, i, size);
}
Server side
using(var file = File.Create("C:\\test\\"+fileName.Substring(0, fileName.IndexOf('\0'))))
while(bytesReceived < numberOfBytes && (count = clientStream.Read(buffer, 0, buffer.Length)) > 0)
{
file.Write(buffer, 0, count);
bytesReceived += count;
}
Here is link on full code - http://pastebin.com/VwTgTxgb

You're doing something very strange here.
First of all, retrieval of file name can be greatly simplified down to Path.GetFileName() call.
Second, are you sure ASCII will suffice?
Third, reading the entire file into memory is OK-ish for a proof-of-concept project, but be ready to switch to streaming operations.
Fourth, your protocol is somewhat wonky. When sending variable-size payload, it is required to first tell the receiving party exactly how much bytes are you going to send. This is exactly what you don't do when sending file name.
Here's a snippet to get you started:
using System;
using System.Diagnostics;
using System.IO;
using System.Net;
using System.Net.Sockets;
using System.Text;
namespace FolderSync
{
class Program
{
static void Main()
{
var server = new Server();
server.Start();
new Client().TransmitFile(
new IPEndPoint(IPAddress.Loopback, 35434),
#"f:\downloads\ubuntu-14.04.3-desktop-amd64.iso");
Console.ReadLine();
server.Stop();
}
}
class Server
{
private readonly TcpListener tcpListener;
public Server()
{
tcpListener = new TcpListener(IPAddress.Loopback, 35434);
}
public void Start()
{
tcpListener.Start();
tcpListener.BeginAcceptTcpClient(AcceptTcpClientCallback, null);
}
public void Stop()
{
tcpListener.Stop();
}
private void AcceptTcpClientCallback(IAsyncResult asyncResult)
{
//
// Big fat warning: http://stackoverflow.com/a/1230266/60188
tcpListener.BeginAcceptTcpClient(AcceptTcpClientCallback, null);
using(var tcpClient = tcpListener.EndAcceptTcpClient(asyncResult))
using(var networkStream = tcpClient.GetStream())
using(var binaryReader = new BinaryReader(networkStream, Encoding.UTF8))
{
var fileName = binaryReader.ReadString();
var length = binaryReader.ReadInt64();
var mib = length / 1024.0 / 1024.0;
Console.WriteLine("Receiving '{0}' ({1:N1} MiB)", fileName, mib);
var stopwatch = Stopwatch.StartNew();
var fullFilePath = Path.Combine(Path.GetTempPath(), fileName);
using(var fileStream = File.Create(fullFilePath))
networkStream.CopyTo(fileStream);
var elapsed = stopwatch.Elapsed;
Console.WriteLine("Received in {0} ({1:N1} MiB/sec)",
elapsed, mib / elapsed.TotalSeconds);
}
}
}
class Client
{
public void TransmitFile(IPEndPoint endPoint, string fileFullPath)
{
if(!File.Exists(fileFullPath)) return;
using(var tcpClient = new TcpClient())
{
tcpClient.Connect(endPoint);
using(var networkStream = tcpClient.GetStream())
using(var binaryWriter = new BinaryWriter(networkStream, Encoding.UTF8))
{
var fileName = Path.GetFileName(fileFullPath);
Debug.Assert(fileName != null, "fileName != null");
//
// BinaryWriter.Write(string) does length-prefixing automatically
binaryWriter.Write(fileName);
using(var fileStream = File.OpenRead(fileFullPath))
{
binaryWriter.Write(fileStream.Length);
fileStream.CopyTo(networkStream);
}
}
}
}
}
}

getting exception from server in c# socket

I have created a simple server and client program in c#. The server will send a string array to the client and the client will display it and the client will sent an id to server and the server will display it. My sample code is given below. When I run them, I get an exception in the server side. I have marked the line in which I am getting the exception.My sample code is given below:
server:
using System;
using System.Collections.Generic;
using System.Net;
using System.Net.Sockets;
using System.IO;
using System.Text;
using System.Xml.Serialization;
namespace server
{
class Program
{
static void Main(string[] args)
{
TcpListener tcpListener = new TcpListener(IPAddress.Any, 1234);
tcpListener.Start();
while (true)
{
TcpClient tcpClient = tcpListener.AcceptTcpClient();
byte[] data = new byte[1024];
NetworkStream ns = tcpClient.GetStream();
string[] arr1 = new string[] { "one", "two", "three" };
var serializer = new XmlSerializer(typeof(string[]));
serializer.Serialize(tcpClient.GetStream(), arr1);
tcpClient.Close();
int recv = ns.Read(data, 0, data.Length); //getting exception in this line
string id = Encoding.ASCII.GetString(data, 0, recv);
Console.WriteLine(id);
}
}
}
}
client :
using System;
using System.Collections.Generic;
using System.Net;
using System.Net.Sockets;
using System.IO;
using System.Text;
using System.Xml.Serialization;
namespace Client
{
class Program
{
static void Main(string[] args)
{
try
{
byte[] data = new byte[1024];
string stringData;
TcpClient tcpClient = new TcpClient("127.0.0.1", 1234);
NetworkStream ns = tcpClient.GetStream();
var serializer = new XmlSerializer(typeof(string[]));
var stringArr = (string[])serializer.Deserialize(tcpClient.GetStream());
foreach (string s in stringArr)
{
Console.WriteLine(s);
}
string input = Console.ReadLine();
ns.Write(Encoding.ASCII.GetBytes(input), 0, input.Length);
ns.Flush();
}
catch (Exception e)
{
Console.Write(e.Message);
}
Console.Read();
}
}
}
The exception I am getting is:
An unhandled exception of type 'System.ObjectDisposedException' occurred in System.dll
Additional information: Cannot access a disposed object.
Is there anything wrong in the code? What will I need to do to avoid this exception?

It seems to me that in your server code you are closing the TCP client before you continue with reading the data. This is why you are getting the exception.
Also, you are trying to read 1024 bytes, but the client does not send 1024 (unless by an unlikely chance that the string that the client sends which is an input from the user is encoded into a 1024 byte array) so the server will simply block and do nothing because it is waiting for the client to send 1024 bytes.
One good way to manage reading and writing data from/to a stream is through the BinaryReader and BinaryWriter classes.
Consider the following code to solve your issue (it contains some improvements to your code):
Server:
class Program
{
static void Main(string[] args)
{
TcpListener tcpListener = new TcpListener(IPAddress.Any, 1234);
tcpListener.Start();
while (true)
{
using (TcpClient tcpClient = tcpListener.AcceptTcpClient())
{
using (var network_stream = tcpClient.GetStream())
{
using (BinaryReader reader = new BinaryReader(network_stream))
{
using (BinaryWriter writer = new BinaryWriter(network_stream))
{
string[] arr1 = new string[] { "one", "two", "three" };
writer.Write(arr1.Length);
foreach (var item in arr1)
{
writer.Write(item);
}
var id = reader.ReadString();
Console.WriteLine("Id: " + id);
}
}
}
}
}
}
}
Client:
class Program
{
static void Main(string[] args)
{
try
{
using (TcpClient tcpClient = new TcpClient("127.0.0.1", 1234))
{
using (NetworkStream network_stream = tcpClient.GetStream())
{
using (BinaryReader reader = new BinaryReader(network_stream))
{
using (BinaryWriter writer = new BinaryWriter(network_stream))
{
int number_of_strings = reader.ReadInt32();
string[] lines = new string[number_of_strings];
for(int i = 0 ; i < number_of_strings ; i++)
{
lines[i] = reader.ReadString(); //Store the strings if you want to
Console.WriteLine(lines[i]);
}
Console.WriteLine("Please input id:");
string id = Console.ReadLine();
writer.Write(id);
}
}
}
}
}
catch (Exception e)
{
Console.Write(e.Message);
}
Console.Read();
}
}

You are using the NetworkStream after you have Closed the TcpClient (which disposed it). See updated while below:
while (true)
{
var tcpClient = tcpListener.AcceptTcpClient();
var data = new byte[1024];
var arr1 = new string[] { "one", "two", "three" };
var serializer = new XmlSerializer(typeof(string[]));
serializer.Serialize(tcpClient.GetStream(), arr1);
var ns = tcpClient.GetStream();
var recv = ns.Read(data, 0, data.Length); //getting exception in this line
var id = Encoding.ASCII.GetString(data, 0, recv);
Console.WriteLine(id);
ns.Close();
tcpClient.Close();
}

tcpClient Implementing Idisposible interface it is better to call dispose method in tcpclient
tcpClient.Dispose();
else
tcpClient.GetStream().Close();
tcpClient.Close();
Closing the client does not close the stream.
var serializer = new XmlSerializer(typeof(string[]));
serializer.Serialize(tcpClient.GetStream(), arr1);
tcpClient.GetStream().Close()
tcpClient.Close();

Writing/reading string through NetworkStream (sockets) for a chat

Sorry if this is hard to understand, trying out C# for the first time.
I am trying to make a simple public 'chat' between clients that are connected to the server. I've tried passing integers to the server and printing them out and everything was fine, however,when I switched to strings, it seems that it can only pass 1 character (because of ns.Write(converted, 0, 1);). If I increase the ns.Write to ns.Write(converted,0,10) everything crashes (both the client and the server) when I enter a message that is less than 10 characters.
Server code:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using System.Net.Sockets;
namespace MultiServeris
{
class Multiserveris
{
static void Main(string[] args)
{
TcpListener ServerSocket = new TcpListener(1000);
ServerSocket.Start();
Console.WriteLine("Server started");
while (true)
{
TcpClient clientSocket = ServerSocket.AcceptTcpClient();
handleClient client = new handleClient();
client.startClient(clientSocket);
}
}
}
public class handleClient
{
TcpClient clientSocket;
public void startClient(TcpClient inClientSocket)
{
this.clientSocket = inClientSocket;
Thread ctThread = new Thread(Chat);
ctThread.Start();
}
private void Chat()
{
byte[] buffer = new byte[10];
while (true)
{
NetworkStream ns = clientSocket.GetStream();
ns.Read(buffer,0,1);
string line = Encoding.UTF8.GetString(buffer);
Console.WriteLine(line);
}
}
}
}
Client code:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Net.Sockets;
namespace Klientas
{
class Klientas
{
static void Main(string[] args)
{
while (true)
{
TcpClient clientSocket = new TcpClient("localhost", 1000);
NetworkStream ns = clientSocket.GetStream();
byte[] buffer = new byte[10];
string str = Console.ReadLine();
byte[] converted = System.Text.Encoding.UTF8.GetBytes(str);
ns.Write(converted, 0, 1);
}
}
}
}

You're best using the BinaryReader/BinaryWriter classes to correctly format and read out data. This removes the need to process it yourself. For example in the client do:
BinaryWriter writer = new BinaryWriter(clientSocket.GetStream());
writer.Write(str);
And in the server:
BinaryReader reader = new BinaryReader(clientSocket.GetStream());
Console.WriteLine(reader.ReadString());

When using BinaryReader or BinaryWriter on the same stream, and you have .NET framework version 4.5 or above, be sure to leave the underlying stream open by using the overload:
using (var w = new BinaryWriter(stream, Encoding.UTF8, true)) {}