We Keep Coding

Duplicate dictionary key on task list

I'm trying to generate a zip file of pdfs asynchronously to speed things up as follows:
var files = new Dictionary<string, byte[]>();
var fileTasks = new List<Task<Library.Models.Helpers.File>>();
foreach (var i in groups)
{
var task = Task.Run(async () =>
{
var fileName = $"{i.Key.Title.Replace('/', '-')} - Records.pdf";
ViewBag.GroupName= i.Key.Title;
var html = await this.RenderViewAsync("~/Views/Report/_UserRecordsReport.cshtml", i.ToList(), true);
return await _fileUtilityService.HtmlToPDF2(html, null, fileName);
});
fileTasks.Add(task);
}
var completedTaskFiles = await Task.WhenAll(fileTasks);
foreach(var item in completedTaskFiles)
{
files.Add($"{item.FileName}", item.FileResult);
}
return _fileUtilityService.GenerateZIP(files);
I'm generating all my html to pdf file tasks and waiting for them to be completed - then trying to synchronously loop through the completed tasks and add them to my dictionary for zipping but I keep getting the following error:
An item with the same key has already been added
There is no duplicate key in the list of items being added.
EDIT - so the current idea is that because its a scoped service, thats why i'm running into thread issues (attached the file utility service for information)
public class FileUtilityService : IFileUtilityService
{
private readonly IHttpClientFactory _clientFactory;
public FileUtilityService(IHttpClientFactory clientFactory)
{
public async Task<byte[]> HtmlToPDF(string html = null, string url = null)
{
try
{
byte[] res = null;
if (html is null && url != null)
{
var client = _clientFactory.CreateClient();
var requestResp = await client.GetAsync(url);
using var sr = new StreamReader(await requestResp.Content.ReadAsStreamAsync());
html = HttpUtility.HtmlDecode(await sr.ReadToEndAsync());
}
using(var ms = new MemoryStream())
{
HtmlConverter.ConvertToPdf(html, ms);
res = ms.ToArray();
}
return res;
}catch(Exception ex)
{
throw ex;
}
}
public async Task<Library.Models.Helpers.File> HtmlToPDF(string html = null, string url = null, string fileName = "")
{
return new Library.Models.Helpers.File() { FileName = fileName, FileResult = await HtmlToPDF(html, url) };
}

c# set timeout for multipartReader.ReadNextSectionAsync()

I'm using C# .net core to read upload data from multipart post user sending multiple files.
How can I prevent use waiting infinite after read last file in
try
{
var cancellationTokenSource = new CancellationTokenSource();
cancellationTokenSource.CancelAfter(1000);
section = await multipartReader.ReadNextSectionAsync(cancellationTokenSource.Token);
}
catch (Exception ex)
{
throw;
}
Altough I've set cancelationToken for 1 second but it goes infinite, and won't go to
next line if I will send another request.
public static async Task<HttpRequest> FromHttpContextAsync(HttpContext httpContext)
{
bool multipart = false;
HttpRequest retVal = new HttpRequest(httpContext);
var sb = new StringBuilder();
var sr = new StreamReader(httpContext.Stream, Encoding.UTF8);
{
var line1 = await sr.ReadLineAsync();
sb.AppendLine(line1);
var Line1Parts = (line1).Split(' ');
retVal.Methode = Line1Parts[0].ToLower();
retVal.RawUrl = System.Net.WebUtility.UrlDecode(Line1Parts[1]).Replace("&", "&");
var urlPart = retVal.RawUrl.Split('?');
retVal.Url = urlPart[0];
if (urlPart.Length > 1)
{
foreach (var part in urlPart[1].Split(new[] { '&' }, StringSplitOptions.RemoveEmptyEntries))
{
var tmp = part.Split(new[] { '=' }, StringSplitOptions.RemoveEmptyEntries);
try
{
retVal.QueryStrings.Add(tmp[0], tmp[1]);
}
catch (Exception ex)
{
}
}
}
string line = await sr.ReadLineAsync();
sb.AppendLine(line);
int contentLength = 0;
while (!string.IsNullOrEmpty(line))
{
var tmp = line.Split(':');
var key = tmp[0].Trim().ToLower();
retVal.Header.Add(new KeyValuePair<string, string>(tmp[0], tmp[1]));
switch (key)
{
case "cookie":
{
foreach (var part in tmp[1].Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries))
{
var pares = part.Split('=');
if (pares.Length == 2)
{
retVal.Cookies.Add(new KeyValuePair<string, string>(pares[0], pares[1]));
}
}
break;
}
case "content-length":
{
contentLength = int.Parse(tmp[1]);
break;
}
}
line = await sr.ReadLineAsync();
sb.AppendLine(line);
}
if (sb.ToString().Contains("Content-Type: multipart/form-data"))
{
string boundary = FindBoundary(sb.ToString());
MultipartReader multipartReader = new MultipartReader(boundary, httpContext.Stream);
var section = await multipartReader.ReadNextSectionAsync();
while (section != null)
{
// process each image
const int chunkSize = 1024;
var buffer = new byte[chunkSize];
var bytesRead = 0;
var fileName = GetFileName(section.ContentDisposition);
using (var stream = new MemoryStream())
{
do
{
try
{
bytesRead = await section.Body.ReadAsync(buffer, 0, buffer.Length);
}
catch (Exception ex)
{
Console.Write(ex);
}
stream.Write(buffer, 0, bytesRead);
} while (bytesRead > 0);
retVal.Files.Add(new Tuple<string, string, byte[]>("", fileName, stream.ToArray()));
}
try
{
var cancellationTokenSource = new CancellationTokenSource();
cancellationTokenSource.CancelAfter(1000);
section = await multipartReader.ReadNextSectionAsync(cancellationTokenSource.Token);
}
catch (Exception ex)
{
throw;
}
}
foreach (var file in retVal.Files)
{
File.WriteAllBytes("d:\\" + file.Item2, file.Item3);
}
}
HttpContext is inline class of this project and this is the source of HttpContext :
public class HttpContext
{
public HttpRequest Request { get; private set; }
public HttpResponse Response { get; private set; }
public Stream Stream { private set; get; }
private HttpContext(Stream networkStream)
{
Stream = networkStream;
}
public async static Task<HttpContext> FromHttpContextAsync(Stream networkStream)
{
var retVal = new HttpContext(networkStream);
retVal.Request = await HttpRequest.FromHttpContextAsync(retVal);
retVal.Response = HttpResponse.FromHttpContext(retVal);
return retVal;
}
}

While the lack of details and context makes trying to reproduce this issue really hard, I suspect the problem here is due to the fact NetworkStreams (used, under the covers, by your MultipartReader instance) do not yet fully support CancellationTokens. In fact, almost every Socket-related operation on .NET Core just checks for the eventually passed CancellationToken upfront - which is useless, in my opinion.
The good news is that the .NET Core team is actively working on this and I believe the issue will be completely solved in .NET Core 3.0:
https://github.com/dotnet/corefx/issues/24430
As a temporary ugly workaround, you can change your code to wait for both a fabricated delay task and your call to ReadNextSectionAsync(), assuming you don't want to re-used that stalled socket / NetworkStream after that:
var timeoutTask = Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
var readNextSectionTask = multipartReader.ReadNextSectionAsync().ConfigureAwait(false);
if (await Task.WhenAny(timeoutTask, readNextSectionTask).ConfigureAwait(false) == timeoutTask)
{
// TODO: Handle the timeout
}
else
{
section = await readNextSectionTask;
}
Additionally, the fact you are not configuring your awaitable may act as a possible deadlock source (it is unclear to me whether you are running this code on ASP.NET Core itself or on some other synchronization context provider). To exclude this possibility, I would suggest to call ConfigureAwait(false) right after your await calls, as you can see on my previous code block.

Bing Speech to Text API - Communicate via websocket in c#

I'm trying to get the Bing Speech API to work in C# via WebSockets. I've looked through the implementation in Javascript here and have been following the protocol instructions here, but I've come up against a complete brick wall. I can't use the existing C# service because I'm running in a Linux container, so I need to use an implementation on .net Core. Annoyingly, the existing service is closed-source!
I can connect to the web socket successfully, but I can't ever get the server to respond to my connection. I'm expecting to receive a turn.start text message from the server, but I get booted off the server as soon as I've sent a few bytes of an audio file. I know the audio file is in the right format because I've got it directly from the C# service sample here.
I feel like I’ve exhausted the options here. The only thing I can think of now is that I’m not sending the audio chunks correctly. Currently, I’m just sending the audio file in consecutive 4096 bytes. I know the first audio message contains the RIFF header which is only 36 bytes, and then I'm just sending this along with the next (4096-36) bytes.
Here is my code in full. You should just be able to run it as a .net core or .net framework console application, and will need an audio file and an API key.
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Net.WebSockets;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace ConsoleApp3
{
class Program
{
static void Main(string[] args)
{
Task.Run(async () =>
{
var bingService = new BingSpeechToTextService();
var audioFilePath = #"FILEPATH GOES HERE";
var authenticationKey = #"BING AUTHENTICATION KEY GOES HERE";
await bingService.RegisterJob(audioFilePath, authenticationKey);
}).Wait();
}
}
public class BingSpeechToTextService
{
/* #region Private Static Methods */
private static async Task Receiving(ClientWebSocket client)
{
var buffer = new byte[128];
while (true)
{
var result = await client.ReceiveAsync(new ArraySegment<byte>(buffer), CancellationToken.None);
var res = Encoding.UTF8.GetString(buffer, 0, result.Count);
if (result.MessageType == WebSocketMessageType.Text)
{
Console.WriteLine(Encoding.UTF8.GetString(buffer, 0, result.Count));
}
else if (result.MessageType == WebSocketMessageType.Close)
{
Console.WriteLine($"Closing ... reason {client.CloseStatusDescription}");
var description = client.CloseStatusDescription;
//await client.CloseOutputAsync(WebSocketCloseStatus.NormalClosure, "", CancellationToken.None);
break;
}
else
{
Console.WriteLine("Other result");
}
}
}
/* #endregion Private Static Methods */
/* #region Public Static Methods */
public static UInt16 ReverseBytes(UInt16 value)
{
return (UInt16)((value & 0xFFU) << 8 | (value & 0xFF00U) >> 8);
}
/* #endregion Public Static Methods */
/* #region Interface: 'Unscrypt.Bing.SpeechToText.Client.Api.IBingSpeechToTextJobService' Methods */
public async Task<int?> RegisterJob(string audioFilePath, string authenticationKeyStr)
{
var authenticationKey = new BingSocketAuthentication(authenticationKeyStr);
var token = authenticationKey.GetAccessToken();
/* #region Connect web socket */
var cws = new ClientWebSocket();
var connectionId = Guid.NewGuid().ToString("N");
var lang = "en-US";
cws.Options.SetRequestHeader("X-ConnectionId", connectionId);
cws.Options.SetRequestHeader("Authorization", "Bearer " + token);
Console.WriteLine("Connecting to web socket.");
var url = $"wss://speech.platform.bing.com/speech/recognition/interactive/cognitiveservices/v1?format=simple&language={lang}";
await cws.ConnectAsync(new Uri(url), new CancellationToken());
Console.WriteLine("Connected.");
/* #endregion*/
/* #region Receiving */
var receiving = Receiving(cws);
/* #endregion*/
/* #region Sending */
var sending = Task.Run(async () =>
{
/* #region Send speech.config */
dynamic speechConfig =
new
{
context = new
{
system = new
{
version = "1.0.00000"
},
os = new
{
platform = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36",
name = "Browser",
version = ""
},
device = new
{
manufacturer = "SpeechSample",
model = "SpeechSample",
version = "1.0.00000"
}
}
};
var requestId = Guid.NewGuid().ToString("N");
var speechConfigJson = JsonConvert.SerializeObject(speechConfig, Formatting.None);
StringBuilder outputBuilder = new StringBuilder();
outputBuilder.Append("path:speech.config\r\n"); //Should this be \r\n
outputBuilder.Append($"x-timestamp:{DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffK")}\r\n");
outputBuilder.Append($"content-type:application/json\r\n");
outputBuilder.Append("\r\n\r\n");
outputBuilder.Append(speechConfigJson);
var strh = outputBuilder.ToString();
var encoded = Encoding.UTF8.GetBytes(outputBuilder.ToString());
var buffer = new ArraySegment<byte>(encoded, 0, encoded.Length);
if (cws.State != WebSocketState.Open) return;
Console.WriteLine("Sending speech.config");
await cws.SendAsync(buffer, WebSocketMessageType.Text, true, new CancellationToken());
Console.WriteLine("Sent.");
/* #endregion*/
/* #region Send audio parts. */
var fileInfo = new FileInfo(audioFilePath);
var streamReader = fileInfo.OpenRead();
for (int cursor = 0; cursor < fileInfo.Length; cursor++)
{
outputBuilder.Clear();
outputBuilder.Append("path:audio\r\n");
outputBuilder.Append($"x-requestid:{requestId}\r\n");
outputBuilder.Append($"x-timestamp:{DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffK")}\r\n");
outputBuilder.Append($"content-type:audio/x-wav");
var headerBytes = Encoding.ASCII.GetBytes(outputBuilder.ToString());
var headerbuffer = new ArraySegment<byte>(headerBytes, 0, headerBytes.Length);
var str = "0x" + (headerBytes.Length).ToString("X");
var headerHeadBytes = BitConverter.GetBytes((UInt16)headerBytes.Length);
var isBigEndian = !BitConverter.IsLittleEndian;
var headerHead = !isBigEndian ? new byte[] { headerHeadBytes[1], headerHeadBytes[0] } : new byte[] { headerHeadBytes[0], headerHeadBytes[1] };
//Audio should be pcm 16kHz, 16bps mono
var byteLen = 8192 - headerBytes.Length - 2;
var fbuff = new byte[byteLen];
streamReader.Read(fbuff, 0, byteLen);
var arr = headerHead.Concat(headerBytes).Concat(fbuff).ToArray();
var arrSeg = new ArraySegment<byte>(arr, 0, arr.Length);
Console.WriteLine($"Sending data from {cursor}");
if (cws.State != WebSocketState.Open) return;
cursor += byteLen;
var end = cursor >= fileInfo.Length;
await cws.SendAsync(arrSeg, WebSocketMessageType.Binary, true, new CancellationToken());
Console.WriteLine("Data sent");
var dt = Encoding.ASCII.GetString(arr);
}
await cws.SendAsync(new ArraySegment<byte>(), WebSocketMessageType.Binary, true, new CancellationToken());
streamReader.Dispose();
/* #endregion*/
{
var startWait = DateTime.UtcNow;
while ((DateTime.UtcNow - startWait).TotalSeconds < 30)
{
await Task.Delay(1);
}
if (cws.State != WebSocketState.Open) return;
}
});
/* #endregion*/
/* #region Wait for tasks to complete */
await Task.WhenAll(sending, receiving);
if (sending.IsFaulted)
{
var err = sending.Exception;
throw err;
}
if (receiving.IsFaulted)
{
var err = receiving.Exception;
throw err;
}
/* #endregion*/
return null;
}
/* #endregion Interface: 'Unscrypt.Bing.SpeechToText.Client.Api.IBingSpeechToTextJobService' Methods */
public class BingSocketAuthentication
{
public static readonly string FetchTokenUri = "https://api.cognitive.microsoft.com/sts/v1.0";
private string subscriptionKey;
private string token;
private Timer accessTokenRenewer;
//Access token expires every 10 minutes. Renew it every 9 minutes.
private const int RefreshTokenDuration = 9;
public BingSocketAuthentication(string subscriptionKey)
{
this.subscriptionKey = subscriptionKey;
this.token = FetchToken(FetchTokenUri, subscriptionKey).Result;
// renew the token on set duration.
accessTokenRenewer = new Timer(new TimerCallback(OnTokenExpiredCallback),
this,
TimeSpan.FromMinutes(RefreshTokenDuration),
TimeSpan.FromMilliseconds(-1));
}
public string GetAccessToken()
{
return this.token;
}
private void RenewAccessToken()
{
this.token = FetchToken(FetchTokenUri, this.subscriptionKey).Result;
Console.WriteLine("Renewed token.");
}
private void OnTokenExpiredCallback(object stateInfo)
{
try
{
RenewAccessToken();
}
catch (Exception ex)
{
Console.WriteLine(string.Format("Failed renewing access token. Details: {0}", ex.Message));
}
finally
{
try
{
accessTokenRenewer.Change(TimeSpan.FromMinutes(RefreshTokenDuration), TimeSpan.FromMilliseconds(-1));
}
catch (Exception ex)
{
Console.WriteLine(string.Format("Failed to reschedule the timer to renew access token. Details: {0}", ex.Message));
}
}
}
private async Task<string> FetchToken(string fetchUri, string subscriptionKey)
{
using (var client = new HttpClient())
{
client.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", subscriptionKey);
UriBuilder uriBuilder = new UriBuilder(fetchUri);
uriBuilder.Path += "/issueToken";
var result = await client.PostAsync(uriBuilder.Uri.AbsoluteUri, null);
Console.WriteLine("Token Uri: {0}", uriBuilder.Uri.AbsoluteUri);
return await result.Content.ReadAsStringAsync();
}
}
}
}
}

I knew it was going to be simple.
After a frustrating few hours of coding, I've found the problem. I've been forgetting to send a request id along with the speech.config call.

Upload to Dropbox not working .NET SDK

I have the following function that uploads files to Drobox and returns shared links to these files.
private async Task<string> Upload(DropboxClient dbx, string localPath, string remotePath)
{
const int ChunkSize = 4096 * 1024;
using (var fileStream = File.Open(localPath, FileMode.Open))
{
if (fileStream.Length <= ChunkSize)
{
WriteMode mode = new WriteMode();
FileMetadata fileMetadata = await dbx.Files.UploadAsync(remotePath, body: fileStream, mode: mode.AsAdd, autorename: true);
//set the expiry date
var existingDoc = await dbx.Files.GetMetadataAsync(remotePath);
if (existingDoc.IsFile)
{
var sharedLink = dbx.Sharing.ListSharedLinksAsync(remotePath);
var settings = new ListSharedLinksArg(remotePath);
ListSharedLinksResult listSharedLinksResult = await dbx.Sharing.ListSharedLinksAsync(remotePath);
if (listSharedLinksResult.Links.Count > 0)
{
return listSharedLinksResult.Links[0].Url;
}
else
{
var settings2 = new SharedLinkSettings(expires: DateTime.Today.AddDays(7));
SharedLinkMetadata sharedLinkMetadata = await dbx.Sharing.CreateSharedLinkWithSettingsAsync(remotePath, settings2);
return sharedLinkMetadata.Url;
}
}
else
{
var settings = new SharedLinkSettings(expires: DateTime.Today.AddDays(7));
SharedLinkMetadata sharedLinkMetadata = await dbx.Sharing.CreateSharedLinkWithSettingsAsync(fileMetadata.PathLower, settings);
return sharedLinkMetadata.Url;
}
}
else
{
await this.ChunkUpload(dbx, remotePath, fileStream, ChunkSize);
}
return "error";
}
}
But it's not working properly, when it gets to the ListSharedLinksAsync function, it stops working witout throwing any error.
I notices that the files that I try to upload are not accessible after it crashes, I get a "used by another proccess error"...
What am I doing wrong?

It looks like you are missing an await on this row, thus causing a deadlock?
var sharedLink = dbx.Sharing.ListSharedLinksAsync(remotePath);
Should be
var sharedLink = await dbx.Sharing.ListSharedLinksAsync(remotePath);

How to wait for multiple async http request

I'd like to ask about how to wait for multiple async http requests.
My code is like this :
public void Convert(XDocument input, out XDocument output)
{
var ns = input.Root.Name.Namespace;
foreach (var element in input.Root.Descendants(ns + "a"))
{
Uri uri = new Uri((string)element.Attribute("href"));
var wc = new WebClient();
wc.OpenReadCompleted += ((sender, e) =>
{
element.Attribute("href").Value = e.Result.ToString();
}
);
wc.OpenReadAsync(uri);
}
//I'd like to wait here until above async requests are all completed.
output = input;
}
Dose anyone know a solution for this?

There is an article by Scott Hanselman in which he describes how to do non blocking requests. Scrolling to the end of it, there is a public Task<bool> ValidateUrlAsync(string url) method.
You could modify it like this (could be more robust about response reading)
public Task<string> GetAsync(string url)
{
var tcs = new TaskCompletionSource<string>();
var request = (HttpWebRequest)WebRequest.Create(url);
try
{
request.BeginGetResponse(iar =>
{
HttpWebResponse response = null;
try
{
response = (HttpWebResponse)request.EndGetResponse(iar);
using(var reader = new StreamReader(response.GetResponseStream()))
{
tcs.SetResult(reader.ReadToEnd());
}
}
catch(Exception exc) { tcs.SetException(exc); }
finally { if (response != null) response.Close(); }
}, null);
}
catch(Exception exc) { tcs.SetException(exc); }
return tsc.Task;
}
So with this in hand, you could then use it like this
var urls=new[]{"url1","url2"};
var tasks = urls.Select(GetAsync).ToArray();
var completed = Task.Factory.ContinueWhenAll(tasks,
completedTasks =>{
foreach(var result in completedTasks.Select(t=>t.Result))
{
Console.WriteLine(result);
}
});
completed.Wait();
//anything that follows gets executed after all urls have finished downloading
Hope this puts you in the right direction.
PS. this is probably as clear as it can get without using async/await

Consider using continuation passing style. If you can restructure your Convert method like this,
public void ConvertAndContinueWith(XDocument input, Action<XDocument> continueWith)
{
var ns = input.Root.Name.Namespace;
var elements = input.Root.Descendants(ns + "a");
int incompleteCount = input.Root.Descendants(ns + "a").Count;
foreach (var element in elements)
{
Uri uri = new Uri((string)element.Attribute("href"));
var wc = new WebClient();
wc.OpenReadCompleted += ((sender, e) =>
{
element.Attribute("href").Value = e.Result.ToString();
if (interlocked.Decrement(ref incompleteCount) == 0)
// This is the final callback, so we can continue executing.
continueWith(input);
}
);
wc.OpenReadAsync(uri);
}
}
You then run that code like this:
XDocument doc = something;
ConvertAndContinueWith(doc, (finishedDocument) => {
// send the completed document to the web client, or whatever you need to do
});