The task i want to accomplish is to create a Web API service in order to upload a file to Azure storage. At the same time, i would like to have a progress indicator that reflects the actual upload progress. After some research and studying i found out two important things:
First is that i have to split the file manually into chunks, and upload them asynchronously using the PutBlockAsync method from Microsoft.WindowsAzure.Storage.dll.
Second, is that i have to receive the file in my Web API service in Streamed mode and not in Buffered mode.
So until now i have the following implementation:
UploadController.cs
using System.Configuration;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
using System.Web.Http;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Blob;
using WebApiFileUploadToAzureStorage.Infrastructure;
using WebApiFileUploadToAzureStorage.Models;
namespace WebApiFileUploadToAzureStorage.Controllers
{
public class UploadController : ApiController
{
[HttpPost]
public async Task<HttpResponseMessage> UploadFile()
{
if (!Request.Content.IsMimeMultipartContent("form-data"))
{
return Request.CreateResponse(HttpStatusCode.UnsupportedMediaType,
new UploadStatus(null, false, "No form data found on request.", string.Empty, string.Empty));
}
var streamProvider = new MultipartAzureBlobStorageProvider(GetAzureStorageContainer());
var result = await Request.Content.ReadAsMultipartAsync(streamProvider);
if (result.FileData.Count < 1)
{
return Request.CreateResponse(HttpStatusCode.BadRequest,
new UploadStatus(null, false, "No files were uploaded.", string.Empty, string.Empty));
}
return Request.CreateResponse(HttpStatusCode.OK);
}
private static CloudBlobContainer GetAzureStorageContainer()
{
var storageConnectionString = ConfigurationManager.AppSettings["AzureBlobStorageConnectionString"];
var storageAccount = CloudStorageAccount.Parse(storageConnectionString);
var blobClient = storageAccount.CreateCloudBlobClient();
blobClient.DefaultRequestOptions.SingleBlobUploadThresholdInBytes = 1024 * 1024;
var container = blobClient.GetContainerReference("photos");
if (container.Exists())
{
return container;
}
container.Create();
container.SetPermissions(new BlobContainerPermissions
{
PublicAccess = BlobContainerPublicAccessType.Container
});
return container;
}
}
}
MultipartAzureBlobStorageProvider.cs
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.WindowsAzure.Storage.Blob;
namespace WebApiFileUploadToAzureStorage.Infrastructure
{
public class MultipartAzureBlobStorageProvider : MultipartFormDataStreamProvider
{
private readonly CloudBlobContainer _blobContainer;
public MultipartAzureBlobStorageProvider(CloudBlobContainer blobContainer) : base(Path.GetTempPath())
{
_blobContainer = blobContainer;
}
public override Task ExecutePostProcessingAsync()
{
const int blockSize = 256 * 1024;
var fileData = FileData.First();
var fileName = Path.GetFileName(fileData.Headers.ContentDisposition.FileName.Trim('"'));
var blob = _blobContainer.GetBlockBlobReference(fileName);
var bytesToUpload = (new FileInfo(fileData.LocalFileName)).Length;
var fileSize = bytesToUpload;
blob.Properties.ContentType = fileData.Headers.ContentType.MediaType;
blob.StreamWriteSizeInBytes = blockSize;
if (bytesToUpload < blockSize)
{
var cancellationToken = new CancellationToken();
using (var fileStream = new FileStream(fileData.LocalFileName, FileMode.Open, FileAccess.ReadWrite))
{
var upload = blob.UploadFromStreamAsync(fileStream, cancellationToken);
Debug.WriteLine($"Status {upload.Status}.");
upload.ContinueWith(task =>
{
Debug.WriteLine($"Status {task.Status}.");
Debug.WriteLine("Upload is over successfully.");
}, TaskContinuationOptions.OnlyOnRanToCompletion);
upload.ContinueWith(task =>
{
Debug.WriteLine($"Status {task.Status}.");
if (task.Exception != null)
{
Debug.WriteLine("Task could not be completed." + task.Exception.InnerException);
}
}, TaskContinuationOptions.OnlyOnFaulted);
upload.Wait(cancellationToken);
}
}
else
{
var blockIds = new List<string>();
var index = 1;
long startPosition = 0;
long bytesUploaded = 0;
do
{
var bytesToRead = Math.Min(blockSize, bytesToUpload);
var blobContents = new byte[bytesToRead];
using (var fileStream = new FileStream(fileData.LocalFileName, FileMode.Open))
{
fileStream.Position = startPosition;
fileStream.Read(blobContents, 0, (int)bytesToRead);
}
var manualResetEvent = new ManualResetEvent(false);
var blockId = Convert.ToBase64String(Encoding.UTF8.GetBytes(index.ToString("d6")));
Debug.WriteLine($"Now uploading block # {index.ToString("d6")}");
blockIds.Add(blockId);
var upload = blob.PutBlockAsync(blockId, new MemoryStream(blobContents), null);
upload.ContinueWith(task =>
{
bytesUploaded += bytesToRead;
bytesToUpload -= bytesToRead;
startPosition += bytesToRead;
index++;
var percentComplete = (double)bytesUploaded / fileSize;
Debug.WriteLine($"Percent complete: {percentComplete.ToString("P")}");
manualResetEvent.Set();
});
manualResetEvent.WaitOne();
} while (bytesToUpload > 0);
Debug.WriteLine("Now committing block list.");
var putBlockList = blob.PutBlockListAsync(blockIds);
putBlockList.ContinueWith(task =>
{
Debug.WriteLine("Blob uploaded completely.");
});
putBlockList.Wait();
}
File.Delete(fileData.LocalFileName);
return base.ExecutePostProcessingAsync();
}
}
}
I also enabled Streamed mode as this blog post suggests. This approach works great, in terms that the file is uploaded successfully to Azure storage. Then, when i make a call to this service making use of XMLHttpRequest (and subscribing to the progress event) i see the indicator moving to 100% very quickly. If a 5MB file needs around 1 minute to upload, my indicator moves to the end in just 1 second. So probably the problem resides in the way that the server informs the client about the upload progress. Any thoughts about this? Thank you.
================================ Update 1 ===================================
That is the JavaScript code i use to call the service
function uploadFile(file, index, uploadCompleted) {
var authData = localStorageService.get("authorizationData");
var xhr = new XMLHttpRequest();
xhr.upload.addEventListener("progress", function (event) {
fileUploadPercent = Math.floor((event.loaded / event.total) * 100);
console.log(fileUploadPercent + " %");
});
xhr.onreadystatechange = function (event) {
if (event.target.readyState === event.target.DONE) {
if (event.target.status !== 200) {
} else {
var parsedResponse = JSON.parse(event.target.response);
uploadCompleted(parsedResponse);
}
}
};
xhr.open("post", uploadFileServiceUrl, true);
xhr.setRequestHeader("Authorization", "Bearer " + authData.token);
var data = new FormData();
data.append("file-" + index, file);
xhr.send(data);
}
your progress indicator might be moving rapidly fast, might be because of
public async Task<HttpResponseMessage> UploadFile()
i have encountered this before, when creating an api of async type, im not even sure if it can be awaited, it will just of course just finish your api call on the background, reason your progress indicator instantly finish, because of the async method (fire and forget). the api will immediately give you a response, but will actually finish on the server background (if not awaited).
please kindly try making it just
public HttpResponseMessage UploadFile()
and also try these ones
var result = Request.Content.ReadAsMultipartAsync(streamProvider).Result;
var upload = blob.UploadFromStreamAsync(fileStream, cancellationToken).Result;
OR
var upload = await blob.UploadFromStreamAsync(fileStream, cancellationToken);
hope it helps.
Other way to acomplish what you want (I don't understand how the XMLHttpRequest's progress event works) is using the ProgressMessageHandler to get the upload progress in the request. Then, in order to notify the client, you could use some cache to store the progress, and from the client request the current state in other endpoint, or use SignalR to send the progress from the server to the client
Something like:
//WebApiConfigRegister
var progress = new ProgressMessageHandler();
progress.HttpSendProgress += HttpSendProgress;
config.MessageHandlers.Add(progress);
//End WebApiConfig Register
private static void HttpSendProgress(object sender, HttpProgressEventArgs e)
{
var request = sender as HttpRequestMessage;
//todo: check if request is not null
//Get an Id from the client or something like this to identify the request
var id = request.RequestUri.Query[0];
var perc = e.ProgressPercentage;
var b = e.TotalBytes;
var bt = e.BytesTransferred;
Cache.InsertOrUpdate(id, perc);
}
You can check more documentation on this MSDN blog post (Scroll down to "Progress Notifications" section)
Also, you could calculate the progress based on the data chunks, store the progress in a cache, and notify in the same way as above. Something like this solution
Related
I have 3000 emails in my gmail account. I want to create an aggregated list of all the senders so that I can more effectively clean up my inbox. I dont need to download the message bodys or the attachments.
I used this sample to get me started (https://developers.google.com/gmail/api/quickstart/dotnet) althought now I cant figure out how to return more than 100 message ids when i execute this code:
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Google.Apis.Auth.OAuth2;
using Google.Apis.Gmail.v1;
using Google.Apis.Gmail.v1.Data;
using Google.Apis.Requests;
using Google.Apis.Services;
using Google.Apis.Util;
using Google.Apis.Util.Store;
namespace GmailQuickstart
{
class Program
{
static string[] Scopes = { GmailService.Scope.GmailReadonly };
static string ApplicationName = "Gmail API .NET Quickstart";
static void Main(string[] args)
{
UserCredential credential;
using (var stream = new FileStream("credentials.json", FileMode.Open, FileAccess.Read))
{
string credPath = "token.json";
credential = GoogleWebAuthorizationBroker.AuthorizeAsync(
GoogleClientSecrets.Load(stream).Secrets,
Scopes,
"user",
CancellationToken.None,
new FileDataStore(credPath, true)).Result;
Console.WriteLine("Credential file saved to: " + credPath);
}
// Create Gmail API service.
var service = new GmailService(new BaseClientService.Initializer()
{
HttpClientInitializer = credential,
ApplicationName = ApplicationName,
});
////get all of the message ids for the messages in the inbox
var messageRequest = service.Users.Messages.List("me");
messageRequest.LabelIds = "INBOX";
var messageList = new List<Message>();
ListMessagesResponse messageResponse1 = new ListMessagesResponse();
var k = 0;
do
{
messageResponse1 = messageRequest.Execute();
messageList.AddRange(messageResponse1.Messages);
var output = $"Request {k} - Message Count: {messageList.Count()} Page Token: {messageRequest.PageToken} - Next Page Token: {messageResponse1.NextPageToken}";
Console.WriteLine(output);
System.IO.File.AppendAllText(#"C:\000\log.txt", output);
messageRequest.PageToken = messageResponse1.NextPageToken;
k++;
//this switch allowed me to walk through getting multiple pages of emails without having to get them all
//if (k == 5)
//{
// break;
//}
} while (!String.IsNullOrEmpty(messageRequest.PageToken));
//once i created the list of all the message ids i serialized the list to JSON and wrote it to a file
//so I could test the next portions without having to make the calls against the above each time
var serializedMessageIdList = Newtonsoft.Json.JsonConvert.SerializeObject(messageList);
System.IO.File.WriteAllText(#"C:\000\MessageIds.json", serializedMessageIdList);
//read in the serialized list and rehydrate it to test the next portion
var mIdList = Newtonsoft.Json.JsonConvert.DeserializeObject<List<Message>>(System.IO.File.ReadAllText(#"C:\000\MessageIds.json"));
//this method takes those message ids and gets the message object from the api for each of them
//1000 is the maximum number of requests google allows in a batch request
var messages = BatchDownloadEmails(service, mIdList.Select(m => m.Id), 1000);
//again i'm serializing the message list and writing them to a file
var serializedMessageList = Newtonsoft.Json.JsonConvert.SerializeObject(messages);
System.IO.File.WriteAllText(#"C:\000\Messages.json", serializedMessageList);
//and then reading them in and rehydrating the list to test the next portion
var mList = Newtonsoft.Json.JsonConvert.DeserializeObject<IList<Message>>(System.IO.File.ReadAllText(#"C:\000\Messages.json"));
//then i loop through each message and pull the values out of the payload header i'm looking for
var emailList = new List<EmailItem>();
foreach (var message in mList)
{
if (message != null)
{
var from = message.Payload.Headers.SingleOrDefault(h => h.Name == "From")?.Value;
var date = message.Payload.Headers.SingleOrDefault(h => h.Name == "Date")?.Value;
var subject = message.Payload.Headers.SingleOrDefault(h => h.Name == "Subject")?.Value;
emailList.Add(new EmailItem() { From = from, Subject = subject, Date = date });
}
}
//i serialized this list as well
var serializedEmailItemList = Newtonsoft.Json.JsonConvert.SerializeObject(emailList);
System.IO.File.WriteAllText(#"C:\000\EmailItems.json", serializedEmailItemList);
//rehydrate for testing
var eiList = Newtonsoft.Json.JsonConvert.DeserializeObject<List<EmailItem>>(System.IO.File.ReadAllText(#"C:\000\EmailItems.json"));
//here is where i do the actual aggregation to determine which senders i have the most email from
var senderSummary = eiList.GroupBy(g => g.From).Select(g => new { Sender = g.Key, Count = g.Count() }).OrderByDescending(g => g.Count);
//serialize and output the results
var serializedSummaryList = Newtonsoft.Json.JsonConvert.SerializeObject(senderSummary);
System.IO.File.WriteAllText(#"C:\000\SenderSummary.json", serializedSummaryList);
}
public static IList<Message> BatchDownloadEmails(GmailService service, IEnumerable<string> messageIds, int chunkSize)
{
// Create a batch request.
var messages = new List<Message>();
//because the google batch request will only allow 1000 requests per batch the list needs to be split
//based on chunk size
var lists = messageIds.ChunkBy(chunkSize);
//double batchRequests = (2500 + 999) / 1000;
//for each list create a request with teh message id and add it to the batch request queue
for (int i = 0; i < lists.Count(); i++)
{
var list = lists.ElementAt(i);
Console.WriteLine($"list: {i}...");
var request = new BatchRequest(service);
foreach (var messageId in list)
{
//Console.WriteLine($"message id: {messageId}...");
var messageBodyRequest = service.Users.Messages.Get("me", messageId);
//messageBodyRequest.Format = UsersResource.MessagesResource.GetRequest.FormatEnum.Metadata;
request.Queue<Message>(messageBodyRequest,
(content, error, index, message) =>
{
messages.Add(content);
});
}
Console.WriteLine("");
Console.WriteLine("ExecuteAsync");
//execute all the requests in the queue
request.ExecuteAsync().Wait();
System.Threading.Thread.Sleep(5000);
}
return messages;
}
}
public class EmailItem
{
public string From { get; set; }
public string Subject { get; set; }
public string Date { get; set; }
}
public static class IEnumerableExtensions
{
public static IEnumerable<IEnumerable<T>> ChunkBy<T>(this IEnumerable<T> source, int chunkSize)
{
return source
.Select((x, i) => new { Index = i, Value = x })
.GroupBy(x => x.Index / chunkSize)
.Select(x => x.Select(v => v.Value));
}
}
}
The research I've done says I need to use a batch request and based on the information I've found Im not able to adapt it to what I'm trying to accomplish. My understanding is that I would use the batch request to get all of the message ids and then 3000 individual calls to get the actual from, subject, and date received from each email in my inbox??
You can use paging to get a full list.
Pass the page token from the previous page to get the next call to Users.Messages.List (don't pass into the first call to get things started). Detect the end when the result contains no messages.
This allows you to get all the messages in the mailbox.
NB. I suggest you make the code async: if there are more than a few messages to read, it can take an appreciable time to get them all.
You can also use PageStreamer to get the remainder of the results.
var pageStreamer = new PageStreamer<Google.Apis.Gmail.v1.Data.Message, UsersResource.MessagesResource.ListRequest, ListMessagesResponse, string>(
(request, token) => request.PageToken = token,
response => response.NextPageToken,
response => response.Messages);
var req = service.Users.Messages.List("me");
req.MaxResults = 1000;
foreach (var result in pageStreamer.Fetch(req))
{
Console.WriteLine(result.Id);
}
This code will continue to run as long as there are additional results to request. Batching isnt really going to help you here as there is no way to know what the next page token will be.
I'm trying to get the Bing Speech API to work in C# via WebSockets. I've looked through the implementation in Javascript here and have been following the protocol instructions here, but I've come up against a complete brick wall. I can't use the existing C# service because I'm running in a Linux container, so I need to use an implementation on .net Core. Annoyingly, the existing service is closed-source!
I can connect to the web socket successfully, but I can't ever get the server to respond to my connection. I'm expecting to receive a turn.start text message from the server, but I get booted off the server as soon as I've sent a few bytes of an audio file. I know the audio file is in the right format because I've got it directly from the C# service sample here.
I feel like I’ve exhausted the options here. The only thing I can think of now is that I’m not sending the audio chunks correctly. Currently, I’m just sending the audio file in consecutive 4096 bytes. I know the first audio message contains the RIFF header which is only 36 bytes, and then I'm just sending this along with the next (4096-36) bytes.
Here is my code in full. You should just be able to run it as a .net core or .net framework console application, and will need an audio file and an API key.
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Net.WebSockets;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace ConsoleApp3
{
class Program
{
static void Main(string[] args)
{
Task.Run(async () =>
{
var bingService = new BingSpeechToTextService();
var audioFilePath = #"FILEPATH GOES HERE";
var authenticationKey = #"BING AUTHENTICATION KEY GOES HERE";
await bingService.RegisterJob(audioFilePath, authenticationKey);
}).Wait();
}
}
public class BingSpeechToTextService
{
/* #region Private Static Methods */
private static async Task Receiving(ClientWebSocket client)
{
var buffer = new byte[128];
while (true)
{
var result = await client.ReceiveAsync(new ArraySegment<byte>(buffer), CancellationToken.None);
var res = Encoding.UTF8.GetString(buffer, 0, result.Count);
if (result.MessageType == WebSocketMessageType.Text)
{
Console.WriteLine(Encoding.UTF8.GetString(buffer, 0, result.Count));
}
else if (result.MessageType == WebSocketMessageType.Close)
{
Console.WriteLine($"Closing ... reason {client.CloseStatusDescription}");
var description = client.CloseStatusDescription;
//await client.CloseOutputAsync(WebSocketCloseStatus.NormalClosure, "", CancellationToken.None);
break;
}
else
{
Console.WriteLine("Other result");
}
}
}
/* #endregion Private Static Methods */
/* #region Public Static Methods */
public static UInt16 ReverseBytes(UInt16 value)
{
return (UInt16)((value & 0xFFU) << 8 | (value & 0xFF00U) >> 8);
}
/* #endregion Public Static Methods */
/* #region Interface: 'Unscrypt.Bing.SpeechToText.Client.Api.IBingSpeechToTextJobService' Methods */
public async Task<int?> RegisterJob(string audioFilePath, string authenticationKeyStr)
{
var authenticationKey = new BingSocketAuthentication(authenticationKeyStr);
var token = authenticationKey.GetAccessToken();
/* #region Connect web socket */
var cws = new ClientWebSocket();
var connectionId = Guid.NewGuid().ToString("N");
var lang = "en-US";
cws.Options.SetRequestHeader("X-ConnectionId", connectionId);
cws.Options.SetRequestHeader("Authorization", "Bearer " + token);
Console.WriteLine("Connecting to web socket.");
var url = $"wss://speech.platform.bing.com/speech/recognition/interactive/cognitiveservices/v1?format=simple&language={lang}";
await cws.ConnectAsync(new Uri(url), new CancellationToken());
Console.WriteLine("Connected.");
/* #endregion*/
/* #region Receiving */
var receiving = Receiving(cws);
/* #endregion*/
/* #region Sending */
var sending = Task.Run(async () =>
{
/* #region Send speech.config */
dynamic speechConfig =
new
{
context = new
{
system = new
{
version = "1.0.00000"
},
os = new
{
platform = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36",
name = "Browser",
version = ""
},
device = new
{
manufacturer = "SpeechSample",
model = "SpeechSample",
version = "1.0.00000"
}
}
};
var requestId = Guid.NewGuid().ToString("N");
var speechConfigJson = JsonConvert.SerializeObject(speechConfig, Formatting.None);
StringBuilder outputBuilder = new StringBuilder();
outputBuilder.Append("path:speech.config\r\n"); //Should this be \r\n
outputBuilder.Append($"x-timestamp:{DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffK")}\r\n");
outputBuilder.Append($"content-type:application/json\r\n");
outputBuilder.Append("\r\n\r\n");
outputBuilder.Append(speechConfigJson);
var strh = outputBuilder.ToString();
var encoded = Encoding.UTF8.GetBytes(outputBuilder.ToString());
var buffer = new ArraySegment<byte>(encoded, 0, encoded.Length);
if (cws.State != WebSocketState.Open) return;
Console.WriteLine("Sending speech.config");
await cws.SendAsync(buffer, WebSocketMessageType.Text, true, new CancellationToken());
Console.WriteLine("Sent.");
/* #endregion*/
/* #region Send audio parts. */
var fileInfo = new FileInfo(audioFilePath);
var streamReader = fileInfo.OpenRead();
for (int cursor = 0; cursor < fileInfo.Length; cursor++)
{
outputBuilder.Clear();
outputBuilder.Append("path:audio\r\n");
outputBuilder.Append($"x-requestid:{requestId}\r\n");
outputBuilder.Append($"x-timestamp:{DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffK")}\r\n");
outputBuilder.Append($"content-type:audio/x-wav");
var headerBytes = Encoding.ASCII.GetBytes(outputBuilder.ToString());
var headerbuffer = new ArraySegment<byte>(headerBytes, 0, headerBytes.Length);
var str = "0x" + (headerBytes.Length).ToString("X");
var headerHeadBytes = BitConverter.GetBytes((UInt16)headerBytes.Length);
var isBigEndian = !BitConverter.IsLittleEndian;
var headerHead = !isBigEndian ? new byte[] { headerHeadBytes[1], headerHeadBytes[0] } : new byte[] { headerHeadBytes[0], headerHeadBytes[1] };
//Audio should be pcm 16kHz, 16bps mono
var byteLen = 8192 - headerBytes.Length - 2;
var fbuff = new byte[byteLen];
streamReader.Read(fbuff, 0, byteLen);
var arr = headerHead.Concat(headerBytes).Concat(fbuff).ToArray();
var arrSeg = new ArraySegment<byte>(arr, 0, arr.Length);
Console.WriteLine($"Sending data from {cursor}");
if (cws.State != WebSocketState.Open) return;
cursor += byteLen;
var end = cursor >= fileInfo.Length;
await cws.SendAsync(arrSeg, WebSocketMessageType.Binary, true, new CancellationToken());
Console.WriteLine("Data sent");
var dt = Encoding.ASCII.GetString(arr);
}
await cws.SendAsync(new ArraySegment<byte>(), WebSocketMessageType.Binary, true, new CancellationToken());
streamReader.Dispose();
/* #endregion*/
{
var startWait = DateTime.UtcNow;
while ((DateTime.UtcNow - startWait).TotalSeconds < 30)
{
await Task.Delay(1);
}
if (cws.State != WebSocketState.Open) return;
}
});
/* #endregion*/
/* #region Wait for tasks to complete */
await Task.WhenAll(sending, receiving);
if (sending.IsFaulted)
{
var err = sending.Exception;
throw err;
}
if (receiving.IsFaulted)
{
var err = receiving.Exception;
throw err;
}
/* #endregion*/
return null;
}
/* #endregion Interface: 'Unscrypt.Bing.SpeechToText.Client.Api.IBingSpeechToTextJobService' Methods */
public class BingSocketAuthentication
{
public static readonly string FetchTokenUri = "https://api.cognitive.microsoft.com/sts/v1.0";
private string subscriptionKey;
private string token;
private Timer accessTokenRenewer;
//Access token expires every 10 minutes. Renew it every 9 minutes.
private const int RefreshTokenDuration = 9;
public BingSocketAuthentication(string subscriptionKey)
{
this.subscriptionKey = subscriptionKey;
this.token = FetchToken(FetchTokenUri, subscriptionKey).Result;
// renew the token on set duration.
accessTokenRenewer = new Timer(new TimerCallback(OnTokenExpiredCallback),
this,
TimeSpan.FromMinutes(RefreshTokenDuration),
TimeSpan.FromMilliseconds(-1));
}
public string GetAccessToken()
{
return this.token;
}
private void RenewAccessToken()
{
this.token = FetchToken(FetchTokenUri, this.subscriptionKey).Result;
Console.WriteLine("Renewed token.");
}
private void OnTokenExpiredCallback(object stateInfo)
{
try
{
RenewAccessToken();
}
catch (Exception ex)
{
Console.WriteLine(string.Format("Failed renewing access token. Details: {0}", ex.Message));
}
finally
{
try
{
accessTokenRenewer.Change(TimeSpan.FromMinutes(RefreshTokenDuration), TimeSpan.FromMilliseconds(-1));
}
catch (Exception ex)
{
Console.WriteLine(string.Format("Failed to reschedule the timer to renew access token. Details: {0}", ex.Message));
}
}
}
private async Task<string> FetchToken(string fetchUri, string subscriptionKey)
{
using (var client = new HttpClient())
{
client.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", subscriptionKey);
UriBuilder uriBuilder = new UriBuilder(fetchUri);
uriBuilder.Path += "/issueToken";
var result = await client.PostAsync(uriBuilder.Uri.AbsoluteUri, null);
Console.WriteLine("Token Uri: {0}", uriBuilder.Uri.AbsoluteUri);
return await result.Content.ReadAsStringAsync();
}
}
}
}
}
I knew it was going to be simple.
After a frustrating few hours of coding, I've found the problem. I've been forgetting to send a request id along with the speech.config call.
How do you delete the dead letters in an Azure Service Bus queue?
I can process the messages in the queue ...
var queueClient = QueueClient.CreateFromConnectionString(sbConnectionString, queueName);
while (queueClient.Peek() != null)
{
var brokeredMessage = queueClient.Receive();
brokeredMessage.Complete();
}
but can't see anyway to handle the dead letter messages
The trick is to get the deadletter path for the queue which you can get by using QueueClient.FormatDeadLetterPath(queueName).
Please try the following:
var queueClient = QueueClient.CreateFromConnectionString(sbConnectionString, QueueClient.FormatDeadLetterPath(queueName));
while (queueClient.Peek() != null)
{
var brokeredMessage = queueClient.Receive();
brokeredMessage.Complete();
}
using System;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Microsoft.Azure.ServiceBus;
using Microsoft.Azure.ServiceBus.Core;
using System.Text;
using System.Collections.Generic;
using System.Threading.Tasks;
namespace ClearDeadLetterQ
{
[TestClass]
public class UnitTest1
{
const string ServiceBusConnectionString = "Endpoint=sb://my-domain.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=youraccesskeyhereyouraccesskeyhere";
[TestMethod]
public async Task TestMethod1()
{
await this.ClearDeadLetters("my.topic.name", "MySubscriptionName/$DeadLetterQueue");
}
public async Task ClearDeadLetters(string topicName, string subscriptionName)
{
var messageReceiver = new MessageReceiver(ServiceBusConnectionString, EntityNameHelper.FormatSubscriptionPath(topicName, subscriptionName), ReceiveMode.PeekLock);
var message = await messageReceiver.ReceiveAsync();
while ((message = await messageReceiver.ReceiveAsync()) != null)
{
await messageReceiver.CompleteAsync(message.SystemProperties.LockToken);
}
await messageReceiver.CloseAsync();
}
}
}
There are some great samples available in our GitHub sample repo (https://github.com/Azure-Samples/azure-servicebus-messaging-samples). The DeadletterQueue project should show you an example of how to do this in your code:
var dead-letterReceiver = await receiverFactory.CreateMessageReceiverAsync(
QueueClient.FormatDeadLetterPath(queueName), ReceiveMode.PeekLock);
while (true)
{
var msg = await dead-letterReceiver.ReceiveAsync(TimeSpan.Zero);
if (msg != null)
{
foreach (var prop in msg.Properties)
{
Console.WriteLine("{0}={1}", prop.Key, prop.Value);
}
await msg.CompleteAsync();
}
else
{
break;
}
}
}
Hope that helps!
Open Azure into your target Bus Queue Service.
(Set this value in place of <BUS-QUEUE-NAME> on the code)
Go into the queue you want to delete.
(Set this value in place of <QUEUE-NAME> on the code)
Create a Shared Access Policy and name it: RemoveDeadLetterQueue with checkbox Manage been selected.
Copy this Primary Key onto <QUEUE-POLICY-PRIMARYKEY> in this code.
And the code is ready to run.
using Microsoft.Azure.ServiceBus.Core;
using Microsoft.Azure.ServiceBus;
string serviceBusQueue = "<BUS-QUEUE-NAME>";
string serviceBusQueueName = "<QUEUE-NAME>";
string policyName = "RemoveDeadLetterQueue";
string policyPrimaryKey = "<QUEUE-POLICY-PRIMARYKEY>";
var receiver = new MessageReceiver(
connectionString: $"Endpoint=sb://{serviceBusQueue}.servicebus.windows.net/;SharedAccessKeyName={policyName};SharedAccessKey={policyPrimaryKey}",
entityPath: $"{serviceBusQueueName}/$DeadLetterQueue",
receiveMode: ReceiveMode.ReceiveAndDelete
);
var messages = await receiver.ReceiveAsync(maxMessageCount: 1000);
while(messages != null)
{
foreach (var message in messages)
{
Console.WriteLine($"[Delete Message] ID: {message.MessageId}");
}
messages = await receiver.ReceiveAsync(maxMessageCount: 1000);
}
await receiver.CloseAsync();
Can someone please provide an example of how to use Google.Apis.Storage.v1 for uploading files to google cloud storage in c#?
I found that this basic operation is not as straight forward as you might expect. Google's documentation about it's Storage API is lacking in information about using it in C# (or any other .NET language). Searching for 'how to upload file to google cloud storage in c#' didn't exactly help me, so here is my working solution with some comments:
Preparation:
You need to create OAuth2 account in your Google Developers Console - go to Project/APIs & auth/Credentials.
Copy Client ID & Client Secret to your code. You will also need your Project name.
Code (it assumes that you've added Google.Apis.Storage.v1 via NuGet):
First, you need to authorize your requests:
var clientSecrets = new ClientSecrets();
clientSecrets.ClientId = clientId;
clientSecrets.ClientSecret = clientSecret;
//there are different scopes, which you can find here https://cloud.google.com/storage/docs/authentication
var scopes = new[] {#"https://www.googleapis.com/auth/devstorage.full_control"};
var cts = new CancellationTokenSource();
var userCredential = await GoogleWebAuthorizationBroker.AuthorizeAsync(clientSecrets,scopes, "yourGoogle#email", cts.Token);
Sometimes you might also want to refresh authorization token via:
await userCredential.RefreshTokenAsync(cts.Token);
You also need to create Storage Service:
var service = new Google.Apis.Storage.v1.StorageService();
Now you can make requests to Google Storage API.
Let's start with creating a new bucket:
var newBucket = new Google.Apis.Storage.v1.Data.Bucket()
{
Name = "your-bucket-name-1"
};
var newBucketQuery = service.Buckets.Insert(newBucket, projectName);
newBucketQuery.OauthToken = userCredential.Result.Token.AccessToken;
//you probably want to wrap this into try..catch block
newBucketQuery.Execute();
And it's done. Now, you can send a request to get list of all of your buckets:
var bucketsQuery = service.Buckets.List(projectName);
bucketsQuery.OauthToken = userCredential.Result.Token.AccessToken;
var buckets = bucketsQuery.Execute();
Last part is uploading new file:
//enter bucket name to which you want to upload file
var bucketToUpload = buckets.Items.FirstOrDefault().Name;
var newObject = new Object()
{
Bucket = bucketToUpload,
Name = "some-file-"+new Random().Next(1,666)
};
FileStream fileStream = null;
try
{
var dir = Directory.GetCurrentDirectory();
var path = Path.Combine(dir, "test.png");
fileStream = new FileStream(path, FileMode.Open);
var uploadRequest = new Google.Apis.Storage.v1.ObjectsResource.InsertMediaUpload(service, newObject,
bucketToUpload,fileStream,"image/png");
uploadRequest.OauthToken = userCredential.Result.Token.AccessToken;
await uploadRequest.UploadAsync();
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
finally
{
if (fileStream != null)
{
fileStream.Dispose();
}
}
And bam! New file will be visible in you Google Developers Console inside of selected bucket.
You can use Google Cloud APIs without SDK in the following ways:
Required api-key.json file
Install package Google.Apis.Auth.OAuth2 in order to authorize the
HTTP web request
You can set the default configuration for your application in this
way
I did the same using .NET core web API and details are given below:
Url details:
"GoogleCloudStorageBaseUrl": "https://www.googleapis.com/upload/storage/v1/b/",
"GoogleSpeechBaseUrl": "https://speech.googleapis.com/v1/operations/",
"GoogleLongRunningRecognizeBaseUrl": "https://speech.googleapis.com/v1/speech:longrunningrecognize",
"GoogleCloudScope": "https://www.googleapis.com/auth/cloud-platform",
public void GetConfiguration()
{
// Set global configuration
bucketName = _configuration.GetValue<string>("BucketName");
googleCloudStorageBaseUrl = _configuration.GetValue<string>("GoogleCloudStorageBaseUrl");
googleSpeechBaseUrl = _configuration.GetValue<string>("GoogleSpeechBaseUrl");
googleLongRunningRecognizeBaseUrl = _configuration.GetValue<string>("GoogleLongRunningRecognizeBaseUrl");
// Set google cloud credentials
string googleApplicationCredentialsPath = _configuration.GetValue<string>("GoogleCloudCredentialPath");
using (Stream stream = new FileStream(googleApplicationCredentialsPath, FileMode.Open, FileAccess.Read))
googleCredential = GoogleCredential.FromStream(stream).CreateScoped(_configuration.GetValue<string>("GoogleCloudScope"));
}
Get Oauth token:
public string GetOAuthToken()
{
return googleCredential.UnderlyingCredential.GetAccessTokenForRequestAsync("https://accounts.google.com/o/oauth2/v2/auth", CancellationToken.None).Result;
}
To upload file to cloud bucket:
public async Task<string> UploadMediaToCloud(string filePath, string objectName = null)
{
string bearerToken = GetOAuthToken();
byte[] fileBytes = File.ReadAllBytes(filePath);
objectName = objectName ?? Path.GetFileName(filePath);
var baseUrl = new Uri(string.Format(googleCloudStorageBaseUrl + "" + bucketName + "/o?uploadType=media&name=" + objectName + ""));
using (WebClient client = new WebClient())
{
client.Headers.Add(HttpRequestHeader.Authorization, "Bearer " + bearerToken);
client.Headers.Add(HttpRequestHeader.ContentType, "application/octet-stream");
byte[] response = await Task.Run(() => client.UploadData(baseUrl, "POST", fileBytes));
string responseInString = Encoding.UTF8.GetString(response);
return responseInString;
}
}
In order to perform any action to the cloud API, just need to make a HttpClient get/post request as per the requirement.
Thanks
This is for Google.Cloud.Storage.V1 (not Google.Apis.Storage.v1), but appears to be a bit simpler to perform an upload now. I started with the Client libraries "Getting Started" instructions to create a service account and bucket, then experimented to find out how to upload an image.
The process I followed was:
Sign up for Google Cloud free trial
Create a new project in Google Cloud (remember the project name\ID for later)
Create a Project Owner service account - this will result in a json file being downloaded that contains the service account credentials. Remember where you put that file.
The getting started docs get you to add the path to the JSON credentials file into an environment variable called GOOGLE_APPLICATION_CREDENTIALS - I couldn't get this to work through the provided instructions. Turns out it is not required, as you can just read the JSON file into a string and pass it to the client constructor.
I created an empty WPF project as a starting point, and a single ViewModel to house the application logic.
Install the Google.Cloud.Storage.V1 nuget package and it should pull in all the dependencies it needs.
Onto the code.
MainWindow.xaml
<StackPanel>
<Button
Margin="50"
Height="50"
Content="BEGIN UPLOAD"
Click="OnButtonClick" />
<ContentControl
Content="{Binding Path=ProgressBar}" />
</StackPanel>
MainWindow.xaml.cs
public partial class MainWindow
{
readonly ViewModel _viewModel;
public MainWindow()
{
_viewModel = new ViewModel(Dispatcher);
DataContext = _viewModel;
InitializeComponent();
}
void OnButtonClick(object sender, RoutedEventArgs args)
{
_viewModel.UploadAsync().ConfigureAwait(false);
}
}
ViewModel.cs
public class ViewModel
{
readonly Dispatcher _dispatcher;
public ViewModel(Dispatcher dispatcher)
{
_dispatcher = dispatcher;
ProgressBar = new ProgressBar {Height=30};
}
public async Task UploadAsync()
{
// Google Cloud Platform project ID.
const string projectId = "project-id-goes-here";
// The name for the new bucket.
const string bucketName = projectId + "-test-bucket";
// Path to the file to upload
const string filePath = #"C:\path\to\image.jpg";
var newObject = new Google.Apis.Storage.v1.Data.Object
{
Bucket = bucketName,
Name = System.IO.Path.GetFileNameWithoutExtension(filePath),
ContentType = "image/jpeg"
};
// read the JSON credential file saved when you created the service account
var credential = Google.Apis.Auth.OAuth2.GoogleCredential.FromJson(System.IO.File.ReadAllText(
#"c:\path\to\service-account-credentials.json"));
// Instantiates a client.
using (var storageClient = Google.Cloud.Storage.V1.StorageClient.Create(credential))
{
try
{
// Creates the new bucket. Only required the first time.
// You can also create buckets through the GCP cloud console web interface
storageClient.CreateBucket(projectId, bucketName);
System.Windows.MessageBox.Show($"Bucket {bucketName} created.");
// Open the image file filestream
using (var fileStream = new System.IO.FileStream(filePath, System.IO.FileMode.Open))
{
ProgressBar.Maximum = fileStream.Length;
// set minimum chunksize just to see progress updating
var uploadObjectOptions = new Google.Cloud.Storage.V1.UploadObjectOptions
{
ChunkSize = Google.Cloud.Storage.V1.UploadObjectOptions.MinimumChunkSize
};
// Hook up the progress callback
var progressReporter = new Progress<Google.Apis.Upload.IUploadProgress>(OnUploadProgress);
await storageClient.UploadObjectAsync(
newObject,
fileStream,
uploadObjectOptions,
progress: progressReporter)
.ConfigureAwait(false);
}
}
catch (Google.GoogleApiException e)
when (e.Error.Code == 409)
{
// When creating the bucket - The bucket already exists. That's fine.
System.Windows.MessageBox.Show(e.Error.Message);
}
catch (Exception e)
{
// other exception
System.Windows.MessageBox.Show(e.Message);
}
}
}
// Called when progress updates
void OnUploadProgress(Google.Apis.Upload.IUploadProgress progress)
{
switch (progress.Status)
{
case Google.Apis.Upload.UploadStatus.Starting:
ProgressBar.Minimum = 0;
ProgressBar.Value = 0;
break;
case Google.Apis.Upload.UploadStatus.Completed:
ProgressBar.Value = ProgressBar.Maximum;
System.Windows.MessageBox.Show("Upload completed");
break;
case Google.Apis.Upload.UploadStatus.Uploading:
UpdateProgressBar(progress.BytesSent);
break;
case Google.Apis.Upload.UploadStatus.Failed:
System.Windows.MessageBox.Show("Upload failed"
+ Environment.NewLine
+ progress.Exception);
break;
}
}
void UpdateProgressBar(long value)
{
_dispatcher.Invoke(() => { ProgressBar.Value = value; });
}
// probably better to expose progress value directly and bind to
// a ProgressBar in the XAML
public ProgressBar ProgressBar { get; }
}
Use of Google.Apis.Storage.v1 for uploading files using SDK to google cloud storage in c#:
Required api-key.json file
Install the package Google.Cloud.Storage.V1; and Google.Apis.Auth.OAuth2;
The code is given below to upload the file to the cloud
private string UploadFile(string localPath, string objectName = null)
{
string projectId = ((Google.Apis.Auth.OAuth2.ServiceAccountCredential)googleCredential.UnderlyingCredential).ProjectId;
try
{
// Creates the new bucket.
var objResult = storageClient.CreateBucket(projectId, bucketName);
if (!string.IsNullOrEmpty(objResult.Id))
{
// Upload file to google cloud server
using (var f = File.OpenRead(localPath))
{
objectName = objectName ?? Path.GetFileName(localPath);
var objFileUploadStatus1 = storageClient.UploadObject(bucketName, objectName, null, f);
}
}
}
catch (Google.GoogleApiException ex)
{
// Error code =409, means bucket already created/exist then upload file in the bucket
if (ex.Error.Code == 409)
{
// Upload file to google cloud server
using (var f = File.OpenRead(localPath))
{
objectName = objectName ?? Path.GetFileName(localPath);
var objFileUploadStatus2 = storageClient.UploadObject(bucketName, objectName, null, f);
}
}
}
catch (Exception ex)
{
MessageBox.Show(ex.ToString());
}
return objectName;
}
To set the credentials
private bool SetStorageCredentials()
{
bool status = true;
try
{
if (File.Exists(credential_path))
{
Environment.SetEnvironmentVariable("GOOGLE_APPLICATION_CREDENTIALS", credential_path);
using (Stream objStream = new FileStream(Environment.GetEnvironmentVariable("GOOGLE_APPLICATION_CREDENTIALS"), FileMode.Open, FileAccess.Read))
googleCredential = GoogleCredential.FromStream(objStream);
// Instantiates a client.
storageClient = StorageClient.Create();
channel = new Grpc.Core.Channel(SpeechClient.DefaultEndpoint.Host, googleCredential.ToChannelCredentials());
}
else
{
DialogResult result = MessageBox.Show("File " + Path.GetFileName(credential_path) + " does not exist. Please provide the correct path.");
if (result == System.Windows.Forms.DialogResult.OK)
{
status = false;
}
}
}
catch (Exception ex)
{
MessageBox.Show(ex.ToString());
status = false;
}
return status;
}
I used SDK in one of my window application. You can use the same code according to your needs/requirements.
You'll be happy to know it still works in 2016...
I was googling all over using fancy key words like "google gcp C# upload image", until I just plain asked the question: "How do I upload an image to google bucket using C#"... and here I am. I removed the .Result in the user credential, and this was the final edit that worked for me.
// ******
static string bucketForImage = ConfigurationManager.AppSettings["testStorageName"];
static string projectName = ConfigurationManager.AppSettings["GCPProjectName"];
string gcpPath = Path.Combine(Server.MapPath("~/Images/Gallery/"), uniqueGcpName + ext);
var clientSecrets = new ClientSecrets();
clientSecrets.ClientId = ConfigurationManager.AppSettings["GCPClientID"];
clientSecrets.ClientSecret = ConfigurationManager.AppSettings["GCPClientSc"];
var scopes = new[] { #"https://www.googleapis.com/auth/devstorage.full_control" };
var cts = new CancellationTokenSource();
var userCredential = await GoogleWebAuthorizationBroker.AuthorizeAsync(clientSecrets, scopes, ConfigurationManager.AppSettings["GCPAccountEmail"], cts.Token);
var service = new Google.Apis.Storage.v1.StorageService();
var bucketToUpload = bucketForImage;
var newObject = new Google.Apis.Storage.v1.Data.Object()
{
Bucket = bucketToUpload,
Name = bkFileName
};
FileStream fileStream = null;
try
{
fileStream = new FileStream(gcpPath, FileMode.Open);
var uploadRequest = new Google.Apis.Storage.v1.ObjectsResource.InsertMediaUpload(service, newObject,
bucketToUpload, fileStream, "image/"+ ext);
uploadRequest.OauthToken = userCredential.Token.AccessToken;
await uploadRequest.UploadAsync();
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
finally
{
if (fileStream != null)
{
fileStream.Dispose();
}
}
// ******
Here is the link to their official C# example of ".NET Bookshelf App" using Google Cloud storage.
https://cloud.google.com/dotnet/docs/getting-started/using-cloud-storage
Source on github:
https://github.com/GoogleCloudPlatform/getting-started-dotnet/blob/master/aspnet/3-binary-data/Services/ImageUploader.cs
https://github.com/GoogleCloudPlatform/getting-started-dotnet/tree/master/aspnet/3-binary-data
Nuget
https://www.nuget.org/packages/Google.Cloud.Storage.V1/
Here are 2 examples that helped me to upload files to a bucket in Google Cloud Storage with Google.Cloud.Storage.V1 (not Google.Apis.Storage.v1):
Upload files to Google cloud storage using c#
Uploading .csv Files to Google Cloud Storage using C# .Net
I got both working on a C# Console Application just for testing purposes.
#February 2021
string _projectId = "YOUR-PROJECT-ID-GCP"; //ProjectID also present in the json file
GoogleCredential _credential = GoogleCredential.FromFile("credential-cloud-file-123418c9e06c.json");
/// <summary>
/// UploadFile to GCS Bucket
/// </summary>
/// <param name="bucketName"></param>
/// <param name="localPath">my-local-path/my-file-name</param>
/// <param name="objectName">my-file-name</param>
public void UploadFile(string bucketName, string localPath, string objectName)
{
var storage = StorageClient.Create(_credential);
using var fileStream = File.OpenRead(localPath);
storage.UploadObject(bucketName, objectName, null, fileStream);
Console.WriteLine($"Uploaded {objectName}.");
}
You get the credentials JSON file from the google cloud portal where you create a bucket under your project..
Simple, with auth:
private void SaveFileToGoogleStorage(string path, string? fileName, string ext)
{
var filePath = Path.Combine(path, fileName + ext);
var gcCredentialsPath = Path.Combine(Environment.CurrentDirectory, "gc_sa_key.json");
Environment.SetEnvironmentVariable("GOOGLE_APPLICATION_CREDENTIALS", gcCredentialsPath);
var gcsStorage = StorageClient.Create();
using var f = File.OpenRead(filePath);
var objectName = Path.GetFileName(filePath);
gcsStorage.UploadObject(_bucketName, objectName, null, f);
Console.WriteLine($"Uploaded {objectName}.");
}
I am trying to make lots of calls from my server to a REST API that is exposed by another server but the code is taking too long to run.
Below is the code which takes a lot of time. Right now I am using C# Async/Await along with HTTPClient, is there a better way I can do this?
static async Task RunAsync()
{
HttpClient client = new HttpClient();
client.BaseAddress = new Uri(URL);
foreach (var workbk in (workbooksforuserresultingMessage.Items[1] as workbookListType).workbook)
{
if (workbk.project.name == "Ascend")
{
tsResponse viewresultingMessage = null;
//Get View Data
HttpRequestMessage viewrequestMessage = new HttpRequestMessage(HttpMethod.Get, "sites/" + ((siteresultingMessage.Items[0] as siteType).id.ToString()) + "/workbooks/" + workbk.id + "/views");
// Add our custom headers
viewrequestMessage.Headers.Add("X-Tableau-Auth", (resultingMessage.Items[0] as credentialsType).token.ToString());
HttpResponseMessage viewrequestMessageresponse = await client.SendAsync(viewrequestMessage);
if (viewrequestMessageresponse.IsSuccessStatusCode)
{
var viewresponsecontent = await viewrequestMessageresponse.Content.ReadAsStringAsync();
XmlSerializer siteserializer = new XmlSerializer(typeof(tsResponse));
using (TextReader reader = new StringReader(viewresponsecontent))
{
viewresultingMessage = (tsResponse)siteserializer.Deserialize(reader);
}
}
}
}
foreach (var workbk in (workbooksforuserresultingMessage.Items[1] as workbookListType).workbook)
{
if (workbk.project.name == "Ascend")
{
foreach (var vu in workbk.views)
{
tsResponse viewImageresultingMessage = null;
//Get View Data
HttpRequestMessage viewImagerequestMessage = new HttpRequestMessage(HttpMethod.Get, "sites/" + ((siteresultingMessage.Items[0] as siteType).id.ToString()) + "/workbooks/" + workbk.id + "/views/" + vu.id + "/previewImage");
// Add our custom headers
viewImagerequestMessage.Headers.Add("X-Tableau-Auth", (resultingMessage.Items[0] as credentialsType).token.ToString());
HttpResponseMessage viewImagewrequestMessageresponse = await client.SendAsync(viewImagerequestMessage);
if (viewImagewrequestMessageresponse.IsSuccessStatusCode)
{
var viewImageresponsecontent = await viewImagewrequestMessageresponse.Content.ReadAsByteArrayAsync();
XmlSerializer siteserializer = new XmlSerializer(typeof(tsResponse));
using (var ms = new MemoryStream(viewImageresponsecontent))
{
Image returnImage = Image.FromStream(ms);
//return returnImage;
}
}
}
}
}
}
Those 'await's inside your loop will serialize the requests. Consider instead using .ContinueWith on the task to do some work with the result, keep track of the tasks in an array, then call Task.WhenAll once you're done setting up all the parallel work.
AS #TheESJ said the awaits make your code execute in a serial fashion. That is once execution reaches the first await, execution is effectively "paused" and will not resume until that Task is complete. You can avoid this by not awaiting the Task, just add it to a list of tasks to keep track of.
To facilitate this I think it will help if you introduced a couple of helper methods to execute the body of your loops and return a Task.
public Task<tsResponse> GetViewString(HttpRequestMessage viewrequestMessage)
{
tsResponse viewresultingMessage = null;
HttpResponseMessage viewrequestMessageresponse = await client.SendAsync(viewrequestMessage);
if (viewrequestMessageresponse.IsSuccessStatusCode)
{
var viewresponsecontent = await viewrequestMessageresponse.Content.ReadAsStringAsync();
XmlSerializer siteserializer = new XmlSerializer(typeof(tsResponse));
using (TextReader reader = new StringReader(viewresponsecontent))
{
viewresultingMessage = (tsResponse)siteserializer.Deserialize(reader);
}
}
return viewresultingMessage;
}
public Task<tsResponse> GetViewImage(HttpRequestMessage viewImagerequestMessage)
{
tsResponse viewImageresultingMessage = null;
//Get View Data
HttpResponseMessage viewImagewrequestMessageresponse = await client.SendAsync(viewImagerequestMessage);
if (viewImagewrequestMessageresponse.IsSuccessStatusCode)
{
var viewImageresponsecontent = await viewImagewrequestMessageresponse.Content.ReadAsByteArrayAsync();
XmlSerializer siteserializer = new XmlSerializer(typeof(tsResponse));
using (var ms = new MemoryStream(viewImageresponsecontent))
{
return Image.FromStream(ms);
}
}
}
Then in your main loop you just add the tasks to a list as the methods are kicked off, this will essentially execute the tasks in parallel. You may need to increase the number of concurrent connections your server can make if it causes a problem. Your main method then becomes.
static async Task RunAsync()
{
HttpClient client = new HttpClient();
client.BaseAddress = new Uri(URL);
var tasks = new List<Task<tsResponse>>();
foreach (var workbk in (workbooksforuserresultingMessage.Items[1] as workbookListType).workbook)
{
if (workbk.project.name == "Ascend")
{
tsResponse viewresultingMessage = null;
//Get View Data
HttpRequestMessage viewrequestMessage = new HttpRequestMessage(HttpMethod.Get, "sites/" + ((siteresultingMessage.Items[0] as siteType).id.ToString()) + "/workbooks/" + workbk.id + "/views");
// Add our custom headers
viewrequestMessage.Headers.Add("X-Tableau-Auth", (resultingMessage.Items[0] as credentialsType).token.ToString());
tasks.Add(GetViewString(viewrequestMessage);
}
}
foreach (var workbk in (workbooksforuserresultingMessage.Items[1] as workbookListType).workbook)
{
if (workbk.project.name == "Ascend")
{
foreach (var vu in workbk.views)
{
tsResponse viewImageresultingMessage = null;
//Get View Data
HttpRequestMessage viewImagerequestMessage = new HttpRequestMessage(HttpMethod.Get, "sites/" + ((siteresultingMessage.Items[0] as siteType).id.ToString()) + "/workbooks/" + workbk.id + "/views/" + vu.id + "/previewImage");
// Add our custom headers
viewImagerequestMessage.Headers.Add("X-Tableau-Auth", (resultingMessage.Items[0] as credentialsType).token.ToString());
tasks.Add(GetViewImage(viewImagerequestMessage);
}
}
}
// wait for all the tasks to complete (non blocking)
await Task.WhenAll(tasks);
}