We Keep Coding

Importing files into SQLite database not using all cores

Context
I am importing IMDB database files into an SQLite database with the help of EntityFrameworkCore. In fact, two files, the titles.basics and the titles.akas (which is linked to basics via its movie ID).
At first, I had a single thread reading lines from basics and loop through akas until it changes of ID. Though, there was an issue there and most of all, it was too slow. So, I decided to create a multithread code that would read both files at the same time and another combining akas with the appropriate movie.
I am currently importing so I still do not know if my issue is fixed (probably it is). Though, it is still too much slow for me.
Issue
The combining part is still very slow, but more importantly, I can see my process is only using around 12% of CPU which corresponds to only 1/8 of total usage and I have 8 physical cores. So, it really seems the process is only using 1 core.
I am not giving any code here, as having a minimal testable code wouldn't mean anything. Though, you can see both versions here:
https://cints.net/public/Imdb-MultiThread.cs.txt
using com.cyberinternauts.all.MediaRecognizer.Database;
using com.cyberinternauts.all.MediaRecognizer.Models.Metas;
using Microsoft.EntityFrameworkCore;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
namespace com.cyberinternauts.all.MediaRecognizer.MetaSources
{
class Imdb : MediaSource
{
private const string TITLES_FILE = "title.basics.tsv.gz";
private const string AKAS_FILE = "title.akas.tsv.gz";
private readonly string temporaryFolder = #"c:\temp\";
private readonly string baseUrl = "https://datasets.imdbws.com/";
private readonly WebClient webClient = new();
MediaRecognizerContext db = new();
private IQueryable<MetaMovie> imdbMovies = null;
private async Task<bool> GatherFilesAsync()
{
var totalFilesGathered = 0;
var filesToDownload = new string[] { AKAS_FILE, TITLES_FILE };
foreach(var fileToDownload in filesToDownload)
{
var compressedFile = temporaryFolder + fileToDownload;
if (!File.Exists(compressedFile) || !File.GetLastWriteTime(compressedFile).Date.Equals(DateTime.Today))
{
await GatherFileAsync(fileToDownload);
totalFilesGathered++;
}
}
return totalFilesGathered != 0;
}
private async Task GatherFileAsync(string fileName)
{
var compressedFile = temporaryFolder + fileName;
var uncompressedFile = temporaryFolder + Path.GetFileNameWithoutExtension(compressedFile);
await webClient.DownloadFileTaskAsync(baseUrl + fileName, compressedFile);
using Stream fd = File.Create(uncompressedFile);
using Stream fs = File.OpenRead(compressedFile);
using Stream csStream = new GZipStream(fs, CompressionMode.Decompress);
var buffer = new byte[1024];
int nRead;
while ((nRead = await csStream.ReadAsync(buffer, 0, buffer.Length)) > 0)
{
await fd.WriteAsync(buffer, 0, nRead);
}
}
private async Task LoadMetaDataAsync()
{
//return; //TODO: Remove this line
//TODO: Reactivate this line
//if (!await GatherFilesAsync()) return;
var titlesFile = temporaryFolder + Path.GetFileNameWithoutExtension(TITLES_FILE);
var akasFile = temporaryFolder + Path.GetFileNameWithoutExtension(AKAS_FILE);
var dbLock = new SemaphoreSlim(1);
var akasLock = new SemaphoreSlim(1);
var currentTitlesAkasLock = new SemaphoreSlim(1);
var associateLock = new SemaphoreSlim(1);
using (var db = new MediaRecognizerContext())
{
db.ChangeTracker.AutoDetectChangesEnabled = false;
var titles = new ConcurrentDictionary<string, MetaMovie>();
var readTitles = Task.Factory.StartNew(() =>
{
Parallel.ForEach(File.ReadLines(titlesFile), (titleLine, _, readingIndex) =>
{
if (readingIndex == 0) return; // Skipping columns titles line
var movieInfos = titleLine.Split("\t", StringSplitOptions.None);
dbLock.Wait();
MetaMovie metaMovie = db.MetaMovies.Where(m => m.ExternalId == movieInfos[0]).Include(m => m.Titles).FirstOrDefault();
dbLock.Release();
if (metaMovie == null)
{
int totalMinutes = -1;
if (!int.TryParse(movieInfos[7], out totalMinutes))
{
totalMinutes = -1;
}
metaMovie = new MetaMovie
{
ExternalId = movieInfos[0],
MetaSource = nameof(Imdb),
MovieType = movieInfos[1],
Title = movieInfos[3],
TotalMinutes = totalMinutes,
Genres = movieInfos[8]
};
metaMovie.Titles = new List<MetaTitle>();
if (int.TryParse(movieInfos[5], out int startYear))
{
metaMovie.StartYear = new DateTime(startYear, 1, 1);
}
else
{
metaMovie.StartYear = new DateTime(9999, 1, 1);
}
if (int.TryParse(movieInfos[6], out int endYear))
{
metaMovie.EndYear = new DateTime(endYear, 1, 1);
}
else
{
metaMovie.EndYear = metaMovie.StartYear;
}
}
titles.TryAdd(metaMovie.ExternalId, metaMovie);
});
});
var akas = new Dictionary<string, List<MetaTitle>>();
var currentTitlesAkas = new ConcurrentDictionary<string, int>();
var readAkas = Task.Factory.StartNew(() =>
{
Parallel.ForEach(File.ReadLines(akasFile), (akaLine, _, readingIndex) =>
{
if (readingIndex == 0) return; // Skipping columns titles line
currentTitlesAkasLock.Wait();
var titleInfos = akaLine.Split("\t", StringSplitOptions.None);
var externalId = titleInfos[0];
if (!currentTitlesAkas.ContainsKey(externalId))
{
currentTitlesAkas.TryAdd(externalId, 1);
}
else
{
currentTitlesAkas[externalId]++;
}
currentTitlesAkasLock.Release();
var metaTitle = new MetaTitle
{
MetaMovie = null,
Text = titleInfos[2],
Region = titleInfos[3],
Language = titleInfos[4]
};
akasLock.Wait();
List<MetaTitle> titleAkas;
if (!akas.ContainsKey(externalId))
{
titleAkas = new List<MetaTitle>();
akas.Add(externalId, titleAkas);
}
else
{
titleAkas = akas[externalId];
}
titleAkas.Add(metaTitle);
akasLock.Release();
currentTitlesAkasLock.Wait();
currentTitlesAkas[externalId]--;
currentTitlesAkasLock.Release();
});
});
var savingCounter = 0;
var associate = Task.Factory.StartNew(() =>
{
Parallel.For(1, Environment.ProcessorCount * 10, async (_) =>
{
var isAssociating = true;
do
{
var externalId = string.Empty;
var currentTitleAkaRemoved = false;
currentTitlesAkasLock.Wait();
foreach (var curExternalId in currentTitlesAkas.Keys.OrderBy(t => t))
{
if (currentTitlesAkas[curExternalId] == 0)
{
externalId = curExternalId;
break;
}
}
if (externalId != String.Empty)
{
currentTitleAkaRemoved = currentTitlesAkas.TryRemove(externalId, out int useless0); // Removing so other threads won't take it
}
isAssociating = !readAkas.IsCompleted || !readTitles.IsCompleted || !currentTitlesAkas.IsEmpty;
currentTitlesAkasLock.Release();
if (String.IsNullOrEmpty(externalId) || !currentTitleAkaRemoved) continue;
if (titles.TryGetValue(externalId, out MetaMovie metaMovie))
{
akasLock.Wait();
var titleAkas = akas[externalId];
akas.Remove(externalId);
akasLock.Release();
var changedMovie = false;
var movieAkas = metaMovie.Titles.Select(t => t).ToList(); // Clone list
foreach (var metaTitle in titleAkas)
{
var existingTitle = movieAkas.Where(t => t.Text == metaTitle.Text && t.Region == metaTitle.Region && t.Language == metaTitle.Language).FirstOrDefault();
if (existingTitle == null)
{
changedMovie = true;
metaMovie.Titles.Add(metaTitle);
}
else
{
movieAkas.Remove(existingTitle);
}
}
foreach (var movieTitle in movieAkas)
{
changedMovie = true;
metaMovie.Titles.Remove(movieTitle);
}
dbLock.Wait();
if (metaMovie.Id == 0)
{
db.Add(metaMovie);
}
else if (changedMovie)
{
db.Update(metaMovie);
}
dbLock.Release();
currentTitlesAkasLock.Wait();
currentTitlesAkas.TryRemove(externalId, out int uselessOut); // Free memory
isAssociating = !readAkas.IsCompleted || !readTitles.IsCompleted || !currentTitlesAkas.IsEmpty;
currentTitlesAkasLock.Release();
titles.TryRemove(externalId, out MetaMovie uselessOut2); // Free memory
associateLock.Wait();
savingCounter++;
var localSavingCounter = savingCounter;
associateLock.Release();
if (localSavingCounter != 0 && localSavingCounter % 1000 == 0)
{
var ttt = currentTitlesAkas.Where(t => t.Value > 0);
dbLock.Wait();
await db.SaveChangesAsync();
dbLock.Release();
Console.WriteLine("Saved " + localSavingCounter);
}
}
else if (!readTitles.IsCompleted) // If reading titles is not ended, then maybe it was not read yet... otherwise, it doesn't exist
{
currentTitlesAkasLock.Wait();
currentTitlesAkas.TryAdd(externalId, 0); // Readd because still no movie associated
currentTitlesAkasLock.Release();
}
} while (isAssociating);
});
});
Task.WaitAll(readTitles, readAkas, associate);
await db.SaveChangesAsync();
}
}
public async override Task<IEnumerable<MetaMovie>> FindMediasAsync(DirectoryInfo directory)
{
await LoadMetaDataAsync();
var movie = await ExtractInfosAsync(directory);
if (movie == null) return null;
if (imdbMovies == null)
{
imdbMovies = db.MetaMovies.Where(m => m.MetaSource == nameof(Imdb) && m.MovieType == "movie");
}
return FindCorrespondances(imdbMovies, movie);
}
}
}
https://cints.net/public/Imdb-SingleThread.cs.txt
using com.cyberinternauts.all.MediaRecognizer.Database;
using com.cyberinternauts.all.MediaRecognizer.Models.Metas;
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
namespace com.cyberinternauts.all.MediaRecognizer.MetaSources
{
class Imdb : MediaSource
{
private const string TITLES_FILE = "title.basics.tsv.gz";
private const string AKAS_FILE = "title.akas.tsv.gz";
private readonly string temporaryFolder = #"c:\temp\";
private readonly string baseUrl = "https://datasets.imdbws.com/";
private readonly WebClient webClient = new();
MediaRecognizerContext db = new();
private IQueryable<MetaMovie> imdbMovies = null;
private async Task<bool> GatherFilesAsync()
{
var totalFilesGathered = 0;
var filesToDownload = new string[] { AKAS_FILE, TITLES_FILE };
foreach(var fileToDownload in filesToDownload)
{
var compressedFile = temporaryFolder + fileToDownload;
if (!File.Exists(compressedFile) || !File.GetLastWriteTime(compressedFile).Date.Equals(DateTime.Today))
{
await GatherFileAsync(fileToDownload);
totalFilesGathered++;
}
}
return totalFilesGathered != 0;
}
private async Task GatherFileAsync(string fileName)
{
var compressedFile = temporaryFolder + fileName;
var uncompressedFile = temporaryFolder + Path.GetFileNameWithoutExtension(compressedFile);
await webClient.DownloadFileTaskAsync(baseUrl + fileName, compressedFile);
using Stream fd = File.Create(uncompressedFile);
using Stream fs = File.OpenRead(compressedFile);
using Stream csStream = new GZipStream(fs, CompressionMode.Decompress);
var buffer = new byte[1024];
int nRead;
while ((nRead = await csStream.ReadAsync(buffer, 0, buffer.Length)) > 0)
{
await fd.WriteAsync(buffer, 0, nRead);
}
}
private async Task LoadMetaDataAsync()
{
//return; //TODO: Remove this line
//TODO: Reactivate this line
//if (!await GatherFilesAsync()) return;
var titlesFile = temporaryFolder + Path.GetFileNameWithoutExtension(TITLES_FILE);
var akasFile = temporaryFolder + Path.GetFileNameWithoutExtension(AKAS_FILE);
var titlesLines = File.ReadLines(titlesFile);
var akasLines = File.ReadLines(akasFile);
var titlesIterator = titlesLines.GetEnumerator();
titlesIterator.MoveNext(); // Skip columns headers
var akasIterator = akasLines.GetEnumerator();
akasIterator.MoveNext();
akasIterator.MoveNext(); // Done twice to skip columns headers
var currentAka = akasIterator.Current;
var savingCounter = 0;
using (var db = new MediaRecognizerContext())
{
db.ChangeTracker.AutoDetectChangesEnabled = false;
while (titlesIterator.MoveNext())
{
var titleLine = titlesIterator.Current;
var movieInfos = titleLine.Split("\t", StringSplitOptions.None);
MetaMovie metaMovie = db.MetaMovies.Where(m => m.ExternalId == movieInfos[0]).FirstOrDefault();
var isNewMovie = false;
if (metaMovie == null)
{
int totalMinutes = -1;
if (!int.TryParse(movieInfos[7], out totalMinutes))
{
totalMinutes = -1;
}
isNewMovie = true;
metaMovie = new MetaMovie
{
ExternalId = movieInfos[0],
MetaSource = nameof(Imdb),
MovieType = movieInfos[1],
Title = movieInfos[3],
TotalMinutes = totalMinutes,
Genres = movieInfos[8]
};
metaMovie.Titles = new List<MetaTitle>();
if (int.TryParse(movieInfos[5], out int startYear))
{
metaMovie.StartYear = new DateTime(startYear, 1, 1);
}
else
{
metaMovie.StartYear = new DateTime(9999, 1, 1);
}
if (int.TryParse(movieInfos[6], out int endYear))
{
metaMovie.EndYear = new DateTime(endYear, 1, 1);
}
else
{
metaMovie.EndYear = metaMovie.StartYear;
}
}
var movieAkasIds = metaMovie.Titles.Select(t => t.Id).ToList();
var titleInfos = currentAka?.Split("\t", StringSplitOptions.None);
while (currentAka != null && int.Parse(titleInfos[0][2..]) <= int.Parse(metaMovie.ExternalId[2..]))
{
if (titleInfos[0] == metaMovie.ExternalId)
{
var metaTitle = new MetaTitle
{
MetaMovie = metaMovie,
Text = titleInfos[2],
Region = titleInfos[3],
Language = titleInfos[4]
};
var existingTitle = metaMovie.Titles.Where(t => t.Text == metaTitle.Text && t.Region == metaTitle.Region && t.Language == metaTitle.Language).FirstOrDefault();
if (existingTitle == null)
{
metaMovie.Titles.Add(metaTitle);
}
else
{
movieAkasIds.Remove(existingTitle.Id);
}
}
else
{
var a = 1;
}
akasIterator.MoveNext();
currentAka = akasIterator.Current;
titleInfos = currentAka.Split("\t", StringSplitOptions.None);
}
foreach(var movieTitleId in movieAkasIds)
{
metaMovie.Titles.Remove(metaMovie.Titles.Where(t => t.Id == movieTitleId).FirstOrDefault());
}
if (isNewMovie)
{
db.Add(metaMovie);
}
else
{
db.Update(metaMovie);
}
savingCounter++;
if (savingCounter % 10000 == 0)
{
await db.SaveChangesAsync();
Console.WriteLine("Saved " + savingCounter);
}
}
await db.SaveChangesAsync();
}
}
public async override Task<IEnumerable<MetaMovie>> FindMediasAsync(DirectoryInfo directory)
{
await LoadMetaDataAsync();
var movie = await ExtractInfosAsync(directory);
if (movie == null) return null;
if (imdbMovies == null)
{
imdbMovies = db.MetaMovies.Where(m => m.MetaSource == nameof(Imdb) && m.MovieType == "movie");
}
return FindCorrespondances(imdbMovies, movie);
}
}
}
In the multithread version, the slow part is in the method LoadMetaDataAsync and more precisely in var associate = Task.Factory.StartNew(() => code part.
This is in development and cleaning, splitting will done after I have the appropriate result/speed.

Case closed. I returned to the single thread version and I found my initial issue (my code was supposing the files were in order, which they were partially).
Thank you for all people that participated.

WebRequests in C# are very CPU Intensive. Need something better

I have a program that needs to scan for other devices running my program on the network. The solution I came up with was to call each ipAddress to see if my program is running.
The code below is completely blocking the cpu:-
using Newtonsoft.Json;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace FileWire
{
class SearchNearby
{
private bool pc_search_cancelled = false;
private SynchronizedCollection<Thread> PCSearchThreadList;
private ConcurrentDictionary<String, String> NearbyPCList;
public void NewPcFound(string s, string s1)
{
Console.WriteLine(string.Format("PC Found at: {0} PC Name: {1}", s, s1));
}
public SearchNearby()
{
startPCScan();
while (true)
{
bool isAnyAlive = false;
foreach(Thread t in PCSearchThreadList)
{
isAnyAlive |= t.IsAlive;
}
if (!isAnyAlive)
{
Console.WriteLine("Search Complete");
foreach (var a in NearbyPCList)
{
Console.WriteLine(a.Key + " ;; " + a.Value);
}
startPCScan();
}
Thread.Sleep(100);
}
}
private void startPCScan()
{
PCSearchThreadList = new SynchronizedCollection<Thread>();
NearbyPCList = new ConcurrentDictionary<String, String>();
pc_search_cancelled = false;
String add = "";
System.Net.IPAddress[] ad = System.Net.Dns.GetHostByName(System.Net.Dns.GetHostName()).AddressList;
foreach (System.Net.IPAddress ip in ad)
{
add += ip.ToString() + "\n";
}
bool connected;
if (add.Trim(' ').Length == 0)
{
connected = false;
}
else
{
connected = true;
}
if (connected)
{
try
{
String[] addresses = add.Split('\n');
foreach (String address in addresses)
{
int myIP = int.Parse(address.Substring(address.LastIndexOf(".") + 1));
for (int def = 0; def <= 10; def++)
{
int finalDef = def;
for (int j = 0; j < 10; j++)
{
string finalJ = j.ToString();
Thread thread = new Thread(new ThreadStart(() =>
{
if (!pc_search_cancelled)
{
for (int i = (finalDef * 25); i < (finalDef * 25) + 25 && i <= 255; i++)
{
if (!pc_search_cancelled)
{
if (i != myIP)
{
String callToAddress = "http://" + address.Substring(0, address.LastIndexOf(".")) + "." + i + ":" + (1234 + int.Parse(finalJ)).ToString();
String name = canGetNameAndAvatar(callToAddress);
if (name != null)
{
NearbyPCList[callToAddress] = name;
NewPcFound(callToAddress, name);
}
}
}
}
}
}));
PCSearchThreadList.Add(thread);
thread.Start();
}
}
}
} catch (Exception e) {
}
}
}
private String canGetNameAndAvatar(String connection)
{
String link = connection + "/getAvatarAndName";
link = link.Replace(" ", "%20");
try
{
var client = new HttpClient();
client.Timeout = TimeSpan.FromMilliseconds(500);
var a = new Task<HttpResponseMessage>[1];
a[0] = client.GetAsync(link);
Task.WaitAll(a);
var b = a[0].Result.Content.ReadAsStringAsync();
Task.WaitAll(b);
Console.WriteLine(b.Result);
string result = b.Result;
result = result.Substring(result.IndexOf("<body>") + 6, result.IndexOf("</body>") - (result.IndexOf("<body>") + 6));
AvtarAndName json = JsonConvert.DeserializeObject<AvtarAndName>(result);
if (json != null)
{
return json.name;
}
}
catch
{
return null;
}
return null;
}
}
}
This is the exact C# version of the java code I was using in Java:-
import com.sun.istack.internal.Nullable;
import org.apache.http.*;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.params.CoreConnectionPNames;
import org.apache.http.params.HttpParams;
import org.json.JSONObject;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.InetAddress;
import java.net.NetworkInterface;
import java.net.SocketException;
import java.net.URI;
import java.util.*;
import java.util.concurrent.CopyOnWriteArrayList;
public class PCScan {
private static boolean pc_search_cancelled = false;
private static List<Thread> PCSearchThreadList;
private static HashMapWithListener<String, String> NearbyPCList;
public static void main(String[] args) {
start();
while (true) {
int numCompleted = 0;
for (Thread t : PCSearchThreadList) {
if (!t.isAlive()) {
numCompleted++;
}
}
if (numCompleted == PCSearchThreadList.size()) {
start();
}
}
}
private static void start() {
try {
startPCScan();
} catch (SocketException e) {
e.printStackTrace();
}
NearbyPCList.setPutListener(new HashMapWithListener.putListener() {
#Override
public void onPut(Object key, Object value) {
System.out.println(key.toString() + ";;" + value.toString());
}
});
}
private static void startPCScan() throws SocketException {
pc_search_cancelled = false;
PCSearchThreadList = new CopyOnWriteArrayList<>();
NearbyPCList = new HashMapWithListener<>();
Enumeration<NetworkInterface> enumeration = NetworkInterface.getNetworkInterfaces();
boolean connected;
String add = "";
while (enumeration.hasMoreElements()) {
NetworkInterface interfacea = enumeration.nextElement();
if (!interfacea.isLoopback()) {
Enumeration<InetAddress> enumeration1 = interfacea.getInetAddresses();
while (enumeration1.hasMoreElements()) {
String address = enumeration1.nextElement().getHostAddress();
if (address.split("\\.").length == 4) {
add += address + "\n";
}
}
}
}
System.out.println(add);
connected = true;
if (connected) {
try {
String[] addresses = add.split("\n");
addresses = new HashSet<String>(Arrays.asList(addresses)).toArray(new String[0]);
for (String address : addresses) {
int myIP = Integer.parseInt(address.substring(address.lastIndexOf(".") + 1));
for (int def = 0; def <= 10; def++) {
int finalDef = def;
for (int j = 0; j < 10; j++) {
int finalJ = j;
Thread thread = new Thread(new Runnable() {
#Override
public void run() {
if (!pc_search_cancelled) {
for (int i = (finalDef * 25); i < (finalDef * 25) + 25 && i <= 255; i++) {
if (!pc_search_cancelled) {
if (i != myIP) {
String callToAddress = "http://" + address.substring(0, address.lastIndexOf(".")) + "." + i + ":" + String.valueOf(Integer.parseInt("1234") + finalJ);
String name = canGetNameAndAvatar(callToAddress);
if (name != null) {
NearbyPCList.put(callToAddress, name);
}
}
}
}
}
}
});
PCSearchThreadList.add(thread);
thread.start();
}
}
// }
// }).start();
}
} catch (Exception e) {
}
}
}
private static String canGetNameAndAvatar(String connection) {
String link = connection + "/getAvatarAndName";
link = link.replaceAll(" ", "%20");
try {
HttpClient client = new DefaultHttpClient();
HttpParams httpParams = client.getParams();
httpParams.setParameter(
CoreConnectionPNames.CONNECTION_TIMEOUT, 500);
HttpGet request = new HttpGet();
request.setURI(new URI(link));
HttpResponse response = client.execute(request);
BufferedReader in = new BufferedReader(new
InputStreamReader(response.getEntity().getContent()));
StringBuffer sb = new StringBuffer("");
String line="";
while ((line = in.readLine()) != null) {
sb.append(line);
break;
}
in.close();
String result = sb.toString();
result = result.substring(result.indexOf("<body>") + 6, result.indexOf("</body>"));
JSONObject json = new JSONObject(result);
if (json != null) {
return json.getString("name");
}
}
catch (Exception ignored){
return null;
}
return null;
}
static class HashMapWithListener<K, V> extends HashMap<K, V> {
private putListener PutListener;
public void setPutListener(putListener PutListener) {
this.PutListener = PutListener;
}
#Nullable
#Override
public V put(K key, V value) {
PutListener.onPut(key, value);
return super.put(key, value);
}
interface putListener {
public void onPut(Object key, Object value);
}
}
}
The java code runs absolutely fine and only uses about 20 percent cpu while c# code absolutely locks the PC. I tried Webclient, webrequest, httpClient. All have literally the same performance.
I need the code to be in c# as I can't include whole JRE in my program since it is too large. The rest of my program and GUI is in WPF format.
Also, I need the code to take a maximum of 50seconds while scanning ports 1234-1243. This code also works absolutely fine even on a midrange android phone. So, I don't know what the problem is.

I would suggest something like this (I've simplified it for the sake of an example):
private static HttpClient _client = new HttpClient() { Timeout = TimeSpan.FromMilliseconds(500) };
private async Task<Something> GetSomething(string url)
{
using (HttpResponseMessage response = await _client.GetAsync(url))
{
string json = await response.ReadAsStringAsync();
return JsonConvert.DeserializeObject<Something>(json);
}
}
private async Task<Something[]> GetSomethings(string[] urls)
{
IEnumerable<Task<Something>> requestTasks = urls.Select(u => GetSomething(u));
Something[] results = await Task.WhenAll<Something>(requestTasks);
return results;
}
You should also make the method calling GetSomethings async, and await it, and do the same all the way up the call chain.
async/await uses a thread pool to execute, and the thread is actually suspended while the IO part of the request occurs, meaning that no CPU time is used during this period. When then IO part is done, it resumes the code at the await.
Related information:
Asynchronous programming documentation
How and when to use 'async' and 'await'

You are using threads and multithreading completely wrong.
Because I cannot really understand what you are trying to do because everything is cramped into one function, I cannot provide you with a more detailed solution.
But let me suggest the following: I understand, you want to execute some operation in the background connecting to some other computer, try something like this
var taskList = new List<Task>();
foreach (var pc in computers)
{
var currentPcTask = Task.Run(() => DoYourWorkForSomePcHere(pc));
taskList.Add(currentPcTask);
}
Task.WaitAll(taskList.ToArray());
This will be very CPU efficient.

Windows 10 UWP VpnPlugin Connection Failures

I am developing a UWP VPN Plugin. In later stage it should handle OpenVPN. In the first stage I am trying to understand the VpnPlugin to get it work in the simplest possible way. For testing I am using Android's ToyVpn test Server on a Debian VM (https://android.googlesource.com/platform/development/+/master/samples/ToyVpn/server/linux). Unfortunately the VpnPlugin is poor and lousy documented, no Guideline - no Nothing. The Github examples are useless and not working either, even https://github.com/ysc3839/UWPToyVpn gives only Rough orientation. I was able to successfully do the Handshake with the Server who responds with a Parameter chain. When it Comes to start the Connection, an exception is thrown that the device is not connected. I am running out of Options and any help would greatly appreciated.
public sealed class ToyVpnPlugin : IVpnPlugIn
{
DatagramSocket _datagramSocket;
public async void Connect(VpnChannel channel)
{
//string parameters = default;
string serverPort = "8000";
string secret = "test";
_datagramSocket = new DatagramSocket();
_datagramSocket.MessageReceived += (s, e) =>
{
DataReader dataReader = e.GetDataReader();
if (dataReader.UnconsumedBufferLength > 0 && dataReader.ReadByte() == 0)
{
var parameters = dataReader.ReadString(dataReader.UnconsumedBufferLength);
ConfigureAndConnect(channel, parameters);
}
};
var serverHostName = channel.Configuration.ServerHostNameList[0];
XmlDocument xmlDocument = new XmlDocument();
xmlDocument.LoadXml(channel.Configuration.CustomField);
var firstChild = xmlDocument.FirstChild;
if (firstChild.Name.Equals("ToyVpnConfig"))
{
foreach (XmlNode childNode in firstChild.ChildNodes)
{
if (childNode.Name.Equals("ServerPort")) serverPort = childNode.InnerText;
else if (childNode.Name.Equals("Secret")) secret = childNode.InnerText;
}
}
await _datagramSocket.ConnectAsync(serverHostName, serverPort);
await HandShake(_datagramSocket, secret);
}
public void Disconnect(VpnChannel channel)
{
channel.Stop();
}
public void GetKeepAlivePayload(VpnChannel channel, out VpnPacketBuffer keepAlivePacket)
{
keepAlivePacket = null;
}
public void Encapsulate(VpnChannel channel, VpnPacketBufferList packets, VpnPacketBufferList encapulatedPackets)
{
while (packets.Size > 0)
{
VpnPacketBuffer vpnPacketBuffer = packets.RemoveAtBegin();
var buffer = vpnPacketBuffer.Buffer;
VpnPacketBufferStatus vpnPacketBufferStatus = vpnPacketBuffer.Status;
encapulatedPackets.Append(vpnPacketBuffer);
}
}
public void Decapsulate(VpnChannel channel, VpnPacketBuffer encapBuffer, VpnPacketBufferList decapsulatedPackets, VpnPacketBufferList controlPacketsToSend)
{
while (encapBuffer != null)
{
decapsulatedPackets.Append(encapBuffer);
}
}
async Task HandShake(DatagramSocket datagramSocket, string secret)
{
for (int i = 0; i < 3; i++)
{
var dataWriter = new DataWriter(datagramSocket.OutputStream)
{
UnicodeEncoding = Windows.Storage.Streams.UnicodeEncoding.Utf8
};
dataWriter.WriteByte(0);
dataWriter.WriteString(secret);
await dataWriter.StoreAsync();
dataWriter.DetachStream();
}
}
void ConfigureAndConnect(VpnChannel vpnChannel, string parameters)
{
parameters = parameters.TrimEnd();
uint mtu = 1500;
List<HostName> ipv4InclusionHostNames = new List<HostName>();
List<HostName> dnsServerHostNames = new List<HostName>();
VpnRouteAssignment vpnRouteAssignment = new VpnRouteAssignment();
var ipv4InclusionRoutes = vpnRouteAssignment.Ipv4InclusionRoutes;
foreach (var parameter in parameters.Split(null))
{
var fields = parameter.Split(",");
try
{
switch (fields[0])
{
case "m":
mtu = uint.Parse(fields[1]);
break;
case "a":
ipv4InclusionHostNames.Add(new HostName(fields[1]));
break;
case "r":
ipv4InclusionRoutes.Add(new VpnRoute(new HostName(fields[1]), (byte)uint.Parse(fields[2])));
break;
case "d":
dnsServerHostNames.Add(new HostName(fields[1]));
break;
case "s":
//TODO "SearchDomain"
break;
default:
break;
}
}
catch (Exception)
{
throw;
}
}
VpnDomainNameAssignment vpnDomainNameAssignment = new VpnDomainNameAssignment();
vpnDomainNameAssignment.DomainNameList.Add(new VpnDomainNameInfo(".", VpnDomainNameType.Suffix, dnsServerHostNames, null));
try
{
vpnChannel.AssociateTransport(_datagramSocket, null);
vpnChannel.StartExistingTransports(ipv4InclusionHostNames, null, null, vpnRouteAssignment, vpnDomainNameAssignment, mtu, 65535, false);
}
catch (Exception e)
{
vpnChannel.TerminateConnection(e.Message);
}
}
}

This appears to be unconfirmed working, any better idea or suggestions are greatly appreciated as I figured this out by more or less "fishing"...:
namespace Background
{
public enum HandshakeState { Waiting, Received, Canceled };
public sealed class SecurepointVpnPlugin : IVpnPlugIn
{
DatagramSocket _datagramSocket;
HandshakeState _handshakeState;
public void Connect(VpnChannel channel)
{
string serverPort = "8000";
string secret = "test";
string parameters = null;
_datagramSocket = new DatagramSocket();
channel.AssociateTransport(_datagramSocket, null);
_datagramSocket.MessageReceived += (s, e) =>
{
DataReader dataReader = e.GetDataReader();
if (dataReader.UnconsumedBufferLength > 0 && dataReader.ReadByte() == 0)
{
parameters = dataReader.ReadString(dataReader.UnconsumedBufferLength);
_handshakeState = HandshakeState.Received;
}
};
var serverHostName = channel.Configuration.ServerHostNameList[0];
XmlDocument xmlDocument = new XmlDocument();
xmlDocument.LoadXml(channel.Configuration.CustomField);
var firstChild = xmlDocument.FirstChild;
if (firstChild.Name.Equals("ToyVpnConfig"))
{
foreach (XmlNode childNode in firstChild.ChildNodes)
{
if (childNode.Name.Equals("ServerPort")) serverPort = childNode.InnerText;
else if (childNode.Name.Equals("Secret")) secret = childNode.InnerText;
}
}
_datagramSocket.ConnectAsync(serverHostName, serverPort).AsTask().GetAwaiter().GetResult();
_handshakeState = HandshakeState.Waiting;
HandShake(_datagramSocket, secret).AsTask().GetAwaiter().GetResult();
if (_handshakeState == HandshakeState.Received) ConfigureAndConnect(channel, parameters);
else channel.Stop();
}
public void Disconnect(VpnChannel channel)
{
channel.Stop();
}
public void GetKeepAlivePayload(VpnChannel channel, out VpnPacketBuffer keepAlivePacket)
{
keepAlivePacket = null;
}
public void Encapsulate(VpnChannel channel, VpnPacketBufferList packets, VpnPacketBufferList encapulatedPackets)
{
while (packets.Size > 0) encapulatedPackets.Append(packets.RemoveAtBegin());
}
public void Decapsulate(VpnChannel channel, VpnPacketBuffer encapBuffer, VpnPacketBufferList decapsulatedPackets, VpnPacketBufferList controlPacketsToSend)
{
decapsulatedPackets.Append(encapBuffer);
}
IAsyncAction HandShake(DatagramSocket datagramSocket, string secret)
{
return Task.Run(async () =>
{
for (int i = 0; i < 3; i++)
{
var dataWriter = new DataWriter(datagramSocket.OutputStream)
{
UnicodeEncoding = Windows.Storage.Streams.UnicodeEncoding.Utf8
};
dataWriter.WriteByte(0);
dataWriter.WriteString(secret);
await dataWriter.StoreAsync();
dataWriter.DetachStream();
}
for (int i = 0; i < 50; i++)
{
await Task.Delay(100);
switch (_handshakeState)
{
case HandshakeState.Waiting:
break;
case HandshakeState.Received:
return;
case HandshakeState.Canceled:
throw new OperationCanceledException();
default:
break;
}
}
}).AsAsyncAction();
}
void ConfigureAndConnect(VpnChannel vpnChannel, string parameters)
{
parameters = parameters.TrimEnd();
uint mtuSize = 68;
var assignedClientIPv4list = new List<HostName>();
var dnsServerList = new List<HostName>();
VpnRouteAssignment assignedRoutes = new VpnRouteAssignment();
VpnDomainNameAssignment assignedDomainName = new VpnDomainNameAssignment();
var ipv4InclusionRoutes = assignedRoutes.Ipv4InclusionRoutes;
foreach (var parameter in parameters.Split(null))
{
var fields = parameter.Split(",");
switch (fields[0])
{
case "m":
mtuSize = uint.Parse(fields[1]);
break;
case "a":
assignedClientIPv4list.Add(new HostName(fields[1]));
break;
case "r":
ipv4InclusionRoutes.Add(new VpnRoute(new HostName(fields[1]), (byte)(int.Parse(fields[2]))));
break;
case "d":
dnsServerList.Add(new HostName(fields[1]));
break;
default:
break;
}
}
assignedRoutes.Ipv4InclusionRoutes = ipv4InclusionRoutes;
assignedDomainName.DomainNameList.Add(new VpnDomainNameInfo(".", VpnDomainNameType.Suffix, dnsServerList, null));
try
{
vpnChannel.StartExistingTransports(assignedClientIPv4list, null, null, assignedRoutes, assignedDomainName, mtuSize, mtuSize + 18, false);
}
catch (Exception e)
{
vpnChannel.TerminateConnection(e.Message);
}
}
}
}

c# set timeout for multipartReader.ReadNextSectionAsync()

I'm using C# .net core to read upload data from multipart post user sending multiple files.
How can I prevent use waiting infinite after read last file in
try
{
var cancellationTokenSource = new CancellationTokenSource();
cancellationTokenSource.CancelAfter(1000);
section = await multipartReader.ReadNextSectionAsync(cancellationTokenSource.Token);
}
catch (Exception ex)
{
throw;
}
Altough I've set cancelationToken for 1 second but it goes infinite, and won't go to
next line if I will send another request.
public static async Task<HttpRequest> FromHttpContextAsync(HttpContext httpContext)
{
bool multipart = false;
HttpRequest retVal = new HttpRequest(httpContext);
var sb = new StringBuilder();
var sr = new StreamReader(httpContext.Stream, Encoding.UTF8);
{
var line1 = await sr.ReadLineAsync();
sb.AppendLine(line1);
var Line1Parts = (line1).Split(' ');
retVal.Methode = Line1Parts[0].ToLower();
retVal.RawUrl = System.Net.WebUtility.UrlDecode(Line1Parts[1]).Replace("&", "&");
var urlPart = retVal.RawUrl.Split('?');
retVal.Url = urlPart[0];
if (urlPart.Length > 1)
{
foreach (var part in urlPart[1].Split(new[] { '&' }, StringSplitOptions.RemoveEmptyEntries))
{
var tmp = part.Split(new[] { '=' }, StringSplitOptions.RemoveEmptyEntries);
try
{
retVal.QueryStrings.Add(tmp[0], tmp[1]);
}
catch (Exception ex)
{
}
}
}
string line = await sr.ReadLineAsync();
sb.AppendLine(line);
int contentLength = 0;
while (!string.IsNullOrEmpty(line))
{
var tmp = line.Split(':');
var key = tmp[0].Trim().ToLower();
retVal.Header.Add(new KeyValuePair<string, string>(tmp[0], tmp[1]));
switch (key)
{
case "cookie":
{
foreach (var part in tmp[1].Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries))
{
var pares = part.Split('=');
if (pares.Length == 2)
{
retVal.Cookies.Add(new KeyValuePair<string, string>(pares[0], pares[1]));
}
}
break;
}
case "content-length":
{
contentLength = int.Parse(tmp[1]);
break;
}
}
line = await sr.ReadLineAsync();
sb.AppendLine(line);
}
if (sb.ToString().Contains("Content-Type: multipart/form-data"))
{
string boundary = FindBoundary(sb.ToString());
MultipartReader multipartReader = new MultipartReader(boundary, httpContext.Stream);
var section = await multipartReader.ReadNextSectionAsync();
while (section != null)
{
// process each image
const int chunkSize = 1024;
var buffer = new byte[chunkSize];
var bytesRead = 0;
var fileName = GetFileName(section.ContentDisposition);
using (var stream = new MemoryStream())
{
do
{
try
{
bytesRead = await section.Body.ReadAsync(buffer, 0, buffer.Length);
}
catch (Exception ex)
{
Console.Write(ex);
}
stream.Write(buffer, 0, bytesRead);
} while (bytesRead > 0);
retVal.Files.Add(new Tuple<string, string, byte[]>("", fileName, stream.ToArray()));
}
try
{
var cancellationTokenSource = new CancellationTokenSource();
cancellationTokenSource.CancelAfter(1000);
section = await multipartReader.ReadNextSectionAsync(cancellationTokenSource.Token);
}
catch (Exception ex)
{
throw;
}
}
foreach (var file in retVal.Files)
{
File.WriteAllBytes("d:\\" + file.Item2, file.Item3);
}
}
HttpContext is inline class of this project and this is the source of HttpContext :
public class HttpContext
{
public HttpRequest Request { get; private set; }
public HttpResponse Response { get; private set; }
public Stream Stream { private set; get; }
private HttpContext(Stream networkStream)
{
Stream = networkStream;
}
public async static Task<HttpContext> FromHttpContextAsync(Stream networkStream)
{
var retVal = new HttpContext(networkStream);
retVal.Request = await HttpRequest.FromHttpContextAsync(retVal);
retVal.Response = HttpResponse.FromHttpContext(retVal);
return retVal;
}
}

While the lack of details and context makes trying to reproduce this issue really hard, I suspect the problem here is due to the fact NetworkStreams (used, under the covers, by your MultipartReader instance) do not yet fully support CancellationTokens. In fact, almost every Socket-related operation on .NET Core just checks for the eventually passed CancellationToken upfront - which is useless, in my opinion.
The good news is that the .NET Core team is actively working on this and I believe the issue will be completely solved in .NET Core 3.0:
https://github.com/dotnet/corefx/issues/24430
As a temporary ugly workaround, you can change your code to wait for both a fabricated delay task and your call to ReadNextSectionAsync(), assuming you don't want to re-used that stalled socket / NetworkStream after that:
var timeoutTask = Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
var readNextSectionTask = multipartReader.ReadNextSectionAsync().ConfigureAwait(false);
if (await Task.WhenAny(timeoutTask, readNextSectionTask).ConfigureAwait(false) == timeoutTask)
{
// TODO: Handle the timeout
}
else
{
section = await readNextSectionTask;
}
Additionally, the fact you are not configuring your awaitable may act as a possible deadlock source (it is unclear to me whether you are running this code on ASP.NET Core itself or on some other synchronization context provider). To exclude this possibility, I would suggest to call ConfigureAwait(false) right after your await calls, as you can see on my previous code block.

Bing Speech to Text API - Communicate via websocket in c#

I'm trying to get the Bing Speech API to work in C# via WebSockets. I've looked through the implementation in Javascript here and have been following the protocol instructions here, but I've come up against a complete brick wall. I can't use the existing C# service because I'm running in a Linux container, so I need to use an implementation on .net Core. Annoyingly, the existing service is closed-source!
I can connect to the web socket successfully, but I can't ever get the server to respond to my connection. I'm expecting to receive a turn.start text message from the server, but I get booted off the server as soon as I've sent a few bytes of an audio file. I know the audio file is in the right format because I've got it directly from the C# service sample here.
I feel like I’ve exhausted the options here. The only thing I can think of now is that I’m not sending the audio chunks correctly. Currently, I’m just sending the audio file in consecutive 4096 bytes. I know the first audio message contains the RIFF header which is only 36 bytes, and then I'm just sending this along with the next (4096-36) bytes.
Here is my code in full. You should just be able to run it as a .net core or .net framework console application, and will need an audio file and an API key.
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Net.WebSockets;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace ConsoleApp3
{
class Program
{
static void Main(string[] args)
{
Task.Run(async () =>
{
var bingService = new BingSpeechToTextService();
var audioFilePath = #"FILEPATH GOES HERE";
var authenticationKey = #"BING AUTHENTICATION KEY GOES HERE";
await bingService.RegisterJob(audioFilePath, authenticationKey);
}).Wait();
}
}
public class BingSpeechToTextService
{
/* #region Private Static Methods */
private static async Task Receiving(ClientWebSocket client)
{
var buffer = new byte[128];
while (true)
{
var result = await client.ReceiveAsync(new ArraySegment<byte>(buffer), CancellationToken.None);
var res = Encoding.UTF8.GetString(buffer, 0, result.Count);
if (result.MessageType == WebSocketMessageType.Text)
{
Console.WriteLine(Encoding.UTF8.GetString(buffer, 0, result.Count));
}
else if (result.MessageType == WebSocketMessageType.Close)
{
Console.WriteLine($"Closing ... reason {client.CloseStatusDescription}");
var description = client.CloseStatusDescription;
//await client.CloseOutputAsync(WebSocketCloseStatus.NormalClosure, "", CancellationToken.None);
break;
}
else
{
Console.WriteLine("Other result");
}
}
}
/* #endregion Private Static Methods */
/* #region Public Static Methods */
public static UInt16 ReverseBytes(UInt16 value)
{
return (UInt16)((value & 0xFFU) << 8 | (value & 0xFF00U) >> 8);
}
/* #endregion Public Static Methods */
/* #region Interface: 'Unscrypt.Bing.SpeechToText.Client.Api.IBingSpeechToTextJobService' Methods */
public async Task<int?> RegisterJob(string audioFilePath, string authenticationKeyStr)
{
var authenticationKey = new BingSocketAuthentication(authenticationKeyStr);
var token = authenticationKey.GetAccessToken();
/* #region Connect web socket */
var cws = new ClientWebSocket();
var connectionId = Guid.NewGuid().ToString("N");
var lang = "en-US";
cws.Options.SetRequestHeader("X-ConnectionId", connectionId);
cws.Options.SetRequestHeader("Authorization", "Bearer " + token);
Console.WriteLine("Connecting to web socket.");
var url = $"wss://speech.platform.bing.com/speech/recognition/interactive/cognitiveservices/v1?format=simple&language={lang}";
await cws.ConnectAsync(new Uri(url), new CancellationToken());
Console.WriteLine("Connected.");
/* #endregion*/
/* #region Receiving */
var receiving = Receiving(cws);
/* #endregion*/
/* #region Sending */
var sending = Task.Run(async () =>
{
/* #region Send speech.config */
dynamic speechConfig =
new
{
context = new
{
system = new
{
version = "1.0.00000"
},
os = new
{
platform = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36",
name = "Browser",
version = ""
},
device = new
{
manufacturer = "SpeechSample",
model = "SpeechSample",
version = "1.0.00000"
}
}
};
var requestId = Guid.NewGuid().ToString("N");
var speechConfigJson = JsonConvert.SerializeObject(speechConfig, Formatting.None);
StringBuilder outputBuilder = new StringBuilder();
outputBuilder.Append("path:speech.config\r\n"); //Should this be \r\n
outputBuilder.Append($"x-timestamp:{DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffK")}\r\n");
outputBuilder.Append($"content-type:application/json\r\n");
outputBuilder.Append("\r\n\r\n");
outputBuilder.Append(speechConfigJson);
var strh = outputBuilder.ToString();
var encoded = Encoding.UTF8.GetBytes(outputBuilder.ToString());
var buffer = new ArraySegment<byte>(encoded, 0, encoded.Length);
if (cws.State != WebSocketState.Open) return;
Console.WriteLine("Sending speech.config");
await cws.SendAsync(buffer, WebSocketMessageType.Text, true, new CancellationToken());
Console.WriteLine("Sent.");
/* #endregion*/
/* #region Send audio parts. */
var fileInfo = new FileInfo(audioFilePath);
var streamReader = fileInfo.OpenRead();
for (int cursor = 0; cursor < fileInfo.Length; cursor++)
{
outputBuilder.Clear();
outputBuilder.Append("path:audio\r\n");
outputBuilder.Append($"x-requestid:{requestId}\r\n");
outputBuilder.Append($"x-timestamp:{DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffK")}\r\n");
outputBuilder.Append($"content-type:audio/x-wav");
var headerBytes = Encoding.ASCII.GetBytes(outputBuilder.ToString());
var headerbuffer = new ArraySegment<byte>(headerBytes, 0, headerBytes.Length);
var str = "0x" + (headerBytes.Length).ToString("X");
var headerHeadBytes = BitConverter.GetBytes((UInt16)headerBytes.Length);
var isBigEndian = !BitConverter.IsLittleEndian;
var headerHead = !isBigEndian ? new byte[] { headerHeadBytes[1], headerHeadBytes[0] } : new byte[] { headerHeadBytes[0], headerHeadBytes[1] };
//Audio should be pcm 16kHz, 16bps mono
var byteLen = 8192 - headerBytes.Length - 2;
var fbuff = new byte[byteLen];
streamReader.Read(fbuff, 0, byteLen);
var arr = headerHead.Concat(headerBytes).Concat(fbuff).ToArray();
var arrSeg = new ArraySegment<byte>(arr, 0, arr.Length);
Console.WriteLine($"Sending data from {cursor}");
if (cws.State != WebSocketState.Open) return;
cursor += byteLen;
var end = cursor >= fileInfo.Length;
await cws.SendAsync(arrSeg, WebSocketMessageType.Binary, true, new CancellationToken());
Console.WriteLine("Data sent");
var dt = Encoding.ASCII.GetString(arr);
}
await cws.SendAsync(new ArraySegment<byte>(), WebSocketMessageType.Binary, true, new CancellationToken());
streamReader.Dispose();
/* #endregion*/
{
var startWait = DateTime.UtcNow;
while ((DateTime.UtcNow - startWait).TotalSeconds < 30)
{
await Task.Delay(1);
}
if (cws.State != WebSocketState.Open) return;
}
});
/* #endregion*/
/* #region Wait for tasks to complete */
await Task.WhenAll(sending, receiving);
if (sending.IsFaulted)
{
var err = sending.Exception;
throw err;
}
if (receiving.IsFaulted)
{
var err = receiving.Exception;
throw err;
}
/* #endregion*/
return null;
}
/* #endregion Interface: 'Unscrypt.Bing.SpeechToText.Client.Api.IBingSpeechToTextJobService' Methods */
public class BingSocketAuthentication
{
public static readonly string FetchTokenUri = "https://api.cognitive.microsoft.com/sts/v1.0";
private string subscriptionKey;
private string token;
private Timer accessTokenRenewer;
//Access token expires every 10 minutes. Renew it every 9 minutes.
private const int RefreshTokenDuration = 9;
public BingSocketAuthentication(string subscriptionKey)
{
this.subscriptionKey = subscriptionKey;
this.token = FetchToken(FetchTokenUri, subscriptionKey).Result;
// renew the token on set duration.
accessTokenRenewer = new Timer(new TimerCallback(OnTokenExpiredCallback),
this,
TimeSpan.FromMinutes(RefreshTokenDuration),
TimeSpan.FromMilliseconds(-1));
}
public string GetAccessToken()
{
return this.token;
}
private void RenewAccessToken()
{
this.token = FetchToken(FetchTokenUri, this.subscriptionKey).Result;
Console.WriteLine("Renewed token.");
}
private void OnTokenExpiredCallback(object stateInfo)
{
try
{
RenewAccessToken();
}
catch (Exception ex)
{
Console.WriteLine(string.Format("Failed renewing access token. Details: {0}", ex.Message));
}
finally
{
try
{
accessTokenRenewer.Change(TimeSpan.FromMinutes(RefreshTokenDuration), TimeSpan.FromMilliseconds(-1));
}
catch (Exception ex)
{
Console.WriteLine(string.Format("Failed to reschedule the timer to renew access token. Details: {0}", ex.Message));
}
}
}
private async Task<string> FetchToken(string fetchUri, string subscriptionKey)
{
using (var client = new HttpClient())
{
client.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", subscriptionKey);
UriBuilder uriBuilder = new UriBuilder(fetchUri);
uriBuilder.Path += "/issueToken";
var result = await client.PostAsync(uriBuilder.Uri.AbsoluteUri, null);
Console.WriteLine("Token Uri: {0}", uriBuilder.Uri.AbsoluteUri);
return await result.Content.ReadAsStringAsync();
}
}
}
}
}

I knew it was going to be simple.
After a frustrating few hours of coding, I've found the problem. I've been forgetting to send a request id along with the speech.config call.