Inside a loop I am creating a task using Task.Run(//...). Each task contains a WebRequest. At Task.WaitAll Result of one task getting overridden by Result of another Task. Whats wrong I am doing ?
When tried with debugger it works fine. Is it because of concurrency ? How to resolve this ?
Below is my code snippet:
SomeMethod()
{
//someItemList.Count() == 5
int r = 0;
Task<MyModel>[] myTaskList= new Task<MyModel>[5];
foreach(var item in someItemList){
Task<MyModel> t = Task<MyModel>.Run(() => { return
SomeOperationWithWebRequest(item); });
myTaskList[r] = t;
r++;
}
Task.WaitAll(myTaskList); //myTaskList[0].Result...myTaskList[4].Result all are having same output.
}
MyModel SomeOperationwithWebRequest(Item){
string URL = "SomeURLFromItem";
string DATA = "DATAfromItem"
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(URL);
request.Method = "POST";
request.ContentType = "application/json";
request.ContentLength = DATA.Length;
using (Stream webStream = request.GetRequestStream())
using (StreamWriter requestWriter = new StreamWriter(webStream, System.Text.Encoding.ASCII))
{
requestWriter.Write(DATA);
}
try
{
WebResponse webResponse = request.GetResponseAsync();
using (Stream webStream = webResponse.GetResponseStream() ?? Stream.Null)
using (StreamReader responseReader = new StreamReader(webStream))
{
//response
}
catch (Exception ex)
{
}
return new MyModel() { // properties
};
}
I think it works in debugging because you dont await the async WebRequest. Try this:
private readonly List<string> _someItemList = new List<string> { "t1", "t2", "t3" };
private async Task SomeMethodAsync()
{
var myTaskList = new List<Task<MyModel>>();
int counter = 0;
foreach (var item in _someItemList)
{
var t = Task.Run(() => SomeOperationWithWebRequestAsync(counter++));
myTaskList.Add(t);
}
await Task.WhenAll(myTaskList);
}
public async Task<MyModel> SomeOperationWithWebRequestAsync(int counter)
{
//do your async request, for simplicity I just do a delay
await Task.Delay(counter * 1000);
return new MyModel {Counter = counter };
}
public class MyModel
{
public int Counter { get; set; }
}
and use it like:
await SomeMethodAsync();
Also notice Task.WhenAll vs Task.WaitAll see e.g
WaitAll vs WhenAll
Related
I need to download numerous web pages' sources. So I need to do that as fast as possible. Here is my codes.
private static async Task<string> downloadsource(string link)
{
ServicePointManager.Expect100Continue = false;
WebRequest req = WebRequest.Create(link);
req.Proxy = null;
req.Method = "GET";
WebResponse res = await siteyeBaglantiTalebi.GetResponseAsync();
StreamReader read = new StreamReader(res.GetResponseStream());
return read.ReadToEnd();
}
List<string> links = new List<string>(){... including some web page links};
private static List<string> source_list(List<string> links)
{
List<string> sources = new List<string>();
for (int i = 0; i < links.Count; i++)
{
Task<string> _task = downloadsource(links[i]);
Console.WriteLine("Downloaded : " + i);
sources.Add(_task.Result);
}
return sources;
}
I was wondering if this code is the fastest way or it can be enhanced.
Can u pls help me with that ?
You are performing a _task.Result call inside each loop. Your code will run as fast as if you downloaded each page one after another if you code it like that.
Try this instead:
private async static Task<List<string>> source_list(List<string> links)
{
List<Task<string>> sources = new List<Task<string>>();
for (int i = 0; i < links.Count; i++)
{
Task<string> _task = downloadsource(links[i]);
Console.WriteLine("Downloading : " + i);
sources.Add(_task);
}
return (await Task.WhenAll(sources)).ToList();
}
This would be even better:
private async static Task<string[]> source_list(List<string> links)
{
return await Task.WhenAll(links.Select(l => downloadsource(l)));
}
Also I cleaned up your downloadsource method:
private static async Task<string> downloadsource(string link)
{
ServicePointManager.Expect100Continue = false;
WebRequest req = WebRequest.Create(link);
req.Proxy = null;
req.Method = "GET";
using (WebResponse res = await req.GetResponseAsync())
{
using (StreamReader read = new StreamReader(res.GetResponseStream()))
{
return read.ReadToEnd();
}
}
}
I have 5 URL and I want to make a Http request for each one, And waiting for the first response that has the conditions.
List<string> urls; // url1, url2, ......
ParallelLoopResult result = Parallel.ForEach(urls, url=> GetTimeSlot(url));
private string GetTimeSlot(string url)
{
HttpWebRequest wr = (HttpWebRequest)WebRequest.Create(url);
HttpWebResponse response = (HttpWebResponse)wr.GetResponse();
string responseString = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding(response.CharacterSet)).ReadToEnd();
if (responseString.Length < 6)
return ""; //PARALEL RESUME
else
return responseString; //PARALEL ENDS
}
I need only the first response. Is it possible with Parallel or is there any better way? Thanks.
Parallel.ForEach will be fine to use especially for your use case.Just use a cancellation token to stop all other running tasks.
static void Main(string[] args)
{
var cts = new CancellationTokenSource();
var _lock = new Object();
var po = new ParallelOptions();
po.CancellationToken = cts.Token;
po.MaxDegreeOfParallelism = System.Environment.ProcessorCount;
var listOfUrls = new List<string>() { "url1", "url2" };
var responsResult = "";
try
{
Parallel.ForEach(listOfUrls, po, (url) =>
{
po.CancellationToken.ThrowIfCancellationRequested();
HttpWebRequest wr = (HttpWebRequest)WebRequest.Create(url);
HttpWebResponse response = (HttpWebResponse)wr.GetResponse();
string responseString = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding(response.CharacterSet)).ReadToEnd();
lock (_lock)
{
if (responseString.Length > 6)
{
responsResult = responseString;
cts.Cancel();
}
}
});
}
catch (OperationCanceledException e)
{
//cancellation was requested
}
finally
{
cts.Dispose();
}
}
You can use PLinq:
string firstResponse = urls
.AsParallel()
.Select(url => GetTimeSlot(url))
.FirstOrDefault(r => ! string.IsNullOrEmpty(r))
;
I am trying to write the following code using HttpClient and async, but it's not able return data.
WebRequest request = WebRequest.Create("some_url");
request.Headers.Add("cookie", "some_cookie");
Stream objStream = request.GetResponse().GetResponseStream();
StreamReader objReader = new StreamReader(objStream);
string sLine = "";
int i = 0;
while (sLine != null)
{
i++;
sLine = objReader.ReadLine();
if (sLine != null)
Console.WriteLine(sLine);
}
Here is what I tried.
using (var client = new HttpClient())
{
client.DefaultRequestHeaders.Add("cookie", "some_cookie");
using (var response = await client.GetAsync("some_url"))
{
string responseData = await response.Content.ReadAsStringAsync();
Console.WriteLine(responseData);
}
}
Any help will be appreciated.
Edit: Here is the code that works for me using HttpClient.
var baseAddress = new Uri(baseUrl);
using (var handler = new HttpClientHandler { UseCookies = false })
using (var client = new HttpClient(handler) { BaseAddress = baseAddress })
{
var requestMessage = new HttpRequestMessage(HttpMethod.Get, queryString);
requestMessage.Headers.Add("cookie", cookie);
var response = client.SendAsync(requestMessage);
response.Wait();
var content = response.Result.Content.ReadAsStringAsync();
content.Wait();
Console.WriteLine(content.Result);
}
Thanks for all the help.
You are doing the async and await, but you are not waiting for response data to be returned or reached so the following snippet will do the job :
Try this code :
static async Task<string> HttpGetResponse()
{
WebRequest request = WebRequest.Create("some_url");
request.Headers.Add("cookie", "some_cookie");
string responseData;
Stream objStream = request.GetResponse().GetResponseStream();
StreamReader objReader = new StreamReader(objStream);
string sLine = "";
int i = 0;
while (sLine != null)
{
i++;
sLine = objReader.ReadLine();
if (sLine != null)
Console.WriteLine(sLine);
}
using (var client = new HttpClient())
{
client.DefaultRequestHeaders.Add("cookie", "some_cookie");
using (var response = await client.GetAsync("some_url"))
{
responseData = await response.Content.ReadAsStringAsync();
Console.WriteLine(responseData);
}
}
return responseData;
}
in main call it like this :
static void Main(string[] args)
{
Task<string> t = HttpGetResponse();
//Do alot of work
t.Wait();
string response = t.Result;
Console.WriteLine(response);
}
Hope this was useful.
This question has been asked a million times, and yet none of the responses work for me. The one I was most excited about was Http Post for Windows Phone 8 but because it requires delegates, it's not right for my code... the Postdata function is called from repositories, it would be nice to get a response straight from this function!
How do I add post parameters to this code? I've been trying to get it to work for a good 10 hours now.
// Repository code
string url = "/bla/bla/" + blaId + "/";
Dictionary<string, string> postParams = new Dictionary<string, string>();
postParams.Add("value", message);
string response = await BlaDataContext.PostData(url, postParams);
// ...
public static async Task<string> PostData(string url, Dictionary<String, String> postParams)
{
HttpWebRequest request = WebRequest.CreateHttp(APIURL + url);
request.ContentType = "application/x-www-form-urlencoded";
request.Method = "POST";
postParams.Add("oauth_token", Contract.AccessToken); // where do I add this to the request??
try
{
HttpWebResponse response = (HttpWebResponse)await request.GetResponseAsync();
Debug.WriteLine(response.ContentType);
Stream responseStream = response.GetResponseStream();
string data;
using (var reader = new StreamReader(responseStream))
{
data = reader.ReadToEnd();
}
responseStream.Close();
return data;
}
catch (Exception e)
{
// whatever
}
}
HttpWebRequest request = WebRequest.CreateHttp("" + url);
//we could move the content-type into a function argument too.
request.ContentType = "application/x-www-form-urlencoded";
request.Method = "POST";
postParams.Add("oauth_token", ""); // where do I add this to the request??
try
{
//this is how you do it
using(var stream = await request.GetRequestStreamAsync())
{
byte[] jsonAsBytes = Encoding.UTF8.GetBytes(string.Join("&", postParams.Select(pp => pp.Key + "=" + pp.Value)));
await stream.WriteAsync(jsonAsBytes, 0, jsonAsBytes.Length);
}
HttpWebResponse response = (HttpWebResponse)await request.GetResponseAsync();
Debug.WriteLine(response.ContentType);
System.IO.Stream responseStream = response.GetResponseStream();
string data;
using (var reader = new System.IO.StreamReader(responseStream))
{
data = reader.ReadToEnd();
}
responseStream.Close();
return data;
}
Async Await HttpWebRequest Extensions:
public static class HttpExtensions
{
public static Task<Stream> GetRequestStreamAsync(this HttpWebRequest request)
{
var tcs = new TaskCompletionSource<Stream>();
try
{
request.BeginGetRequestStream(iar =>
{
try
{
var response = request.EndGetRequestStream(iar);
tcs.SetResult(response);
}
catch (Exception exc)
{
tcs.SetException(exc);
}
}, null);
}
catch (Exception exc)
{
tcs.SetException(exc);
}
return tcs.Task;
}
public static Task<HttpWebResponse> GetResponseAsync(this HttpWebRequest request)
{
var taskComplete = new TaskCompletionSource<HttpWebResponse>();
request.BeginGetResponse(asyncResponse =>
{
try
{
HttpWebRequest responseRequest = (HttpWebRequest)asyncResponse.AsyncState;
HttpWebResponse someResponse =
(HttpWebResponse)responseRequest.EndGetResponse(asyncResponse);
taskComplete.TrySetResult(someResponse);
}
catch (WebException webExc)
{
HttpWebResponse failedResponse = (HttpWebResponse)webExc.Response;
taskComplete.TrySetResult(failedResponse);
}
}, request);
return taskComplete.Task;
}
}
With the extensions, I think it's a little cleaner.
You need to write the parameters to the request body.
You could use an extension method like this one:
public static void AddFormData(this HttpWebRequest request, IDictionary<string, string> data)
{
using (var memStream = new MemoryStream())
using (var writer = new StreamWriter(memStream))
{
bool first = true;
foreach (var d in data)
{
if (!first)
writer.Append("&");
writer.Write(Uri.EscapeDataString(d.Key));
writer.Write("=");
writer.Write(Uri.EscapeDataString(d.Value));
first = false;
}
writer.Flush();
request.ContentLength = memStream.Length;
memStream.Position = 0;
using (var reqStream = request.GetRequestStream())
{
memStream.CopyTo(reqStream);
}
}
}
Call it like this:
request.AddFormData(postParams);
I am having a hard time convert the below code which i have created in 4.0 to 4.5 using HttpClient.
According to my understand i guess if i create multiple web requests in the GUI thread itself without blocking the GUI if i got with asynchronous requeest.
how to convert the below code to Asynchronous using HttpClient in 4.5
// This is what called when button is clicked
Task t3 = new Task(SpawnTask);
t3.Start();
//if noofthreads are less 50 then GUI is woking fine.. if number increases then takes much time for repaint..
//where as other softwares are working without any problem even if the threads are more than 500!! in the same system
public void SpawnTask()
{
try
{
ParallelOptions po = new ParallelOptions();
po.CancellationToken = cts.Token;
po.MaxDegreeOfParallelism = noofthreads;
Parallel.ForEach(
urls,
po,
url => checkpl(url));
}
catch (Exception ex)
{
}
}
public void checkpl(string url)
{
try
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
request.Timeout = 60*1000;
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
string stext = "";
using (BufferedStream buffer = new BufferedStream(response.GetResponseStream()))
{
using (StreamReader reader = new StreamReader(buffer))
{
stext = reader.ReadToEnd();
}
}
response.Close();
if (stext .IndexOf("domainname.com") != -1)
{
tfound = tfound + 1;
string lext = "Total Found : "+tfound.ToString();
label3.BeginInvoke(new InvokeDelegate(UpdateLabel), ltext);
slist.Add(url);
textBox2.BeginInvoke(new InvokeDelegate4(UpdateText), "Working Url " + url);
}
}
catch (Exception ex)
{
}
}
Since you are using .NET 4.5 you can use the new async and await keywords. Here is what it might look like.
private async void YourButton_Click(object sender, EventArgs args)
{
YourButton.Enabled = false;
try
{
var tasks = new List<Task>();
foreach (string url in Urls)
{
tasks.Add(CheckAsync(url));
}
await TaskEx.WhenAll(tasks);
}
finally
{
YourButton.Enabled = true;
}
}
private async Task CheckAsync(string url)
{
bool found = await UrlResponseContainsAsync(url, "domainname.com");
if (found)
{
slist.Add(url);
label3.Text = "Total Found: " + slist.Count.ToString();
textbox2.Text = "Working Url " + url;
}
}
private async Task<bool> UrlResponseContainsAsync(string url, string find)
{
var request = WebRequest.Create(url);
request.Timeout = 60 * 1000;
using (WebResponse response = await request.GetResponseAsync())
{
using (var buffer = new BufferedStream(response.GetResponseStream()))
using (var reader = new StreamReader(buffer))
{
string text = reader.ReadToEnd();
return text.Contains(find);
}
}
}