I have 5 URL and I want to make a Http request for each one, And waiting for the first response that has the conditions.
List<string> urls; // url1, url2, ......
ParallelLoopResult result = Parallel.ForEach(urls, url=> GetTimeSlot(url));
private string GetTimeSlot(string url)
{
HttpWebRequest wr = (HttpWebRequest)WebRequest.Create(url);
HttpWebResponse response = (HttpWebResponse)wr.GetResponse();
string responseString = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding(response.CharacterSet)).ReadToEnd();
if (responseString.Length < 6)
return ""; //PARALEL RESUME
else
return responseString; //PARALEL ENDS
}
I need only the first response. Is it possible with Parallel or is there any better way? Thanks.
Parallel.ForEach will be fine to use especially for your use case.Just use a cancellation token to stop all other running tasks.
static void Main(string[] args)
{
var cts = new CancellationTokenSource();
var _lock = new Object();
var po = new ParallelOptions();
po.CancellationToken = cts.Token;
po.MaxDegreeOfParallelism = System.Environment.ProcessorCount;
var listOfUrls = new List<string>() { "url1", "url2" };
var responsResult = "";
try
{
Parallel.ForEach(listOfUrls, po, (url) =>
{
po.CancellationToken.ThrowIfCancellationRequested();
HttpWebRequest wr = (HttpWebRequest)WebRequest.Create(url);
HttpWebResponse response = (HttpWebResponse)wr.GetResponse();
string responseString = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding(response.CharacterSet)).ReadToEnd();
lock (_lock)
{
if (responseString.Length > 6)
{
responsResult = responseString;
cts.Cancel();
}
}
});
}
catch (OperationCanceledException e)
{
//cancellation was requested
}
finally
{
cts.Dispose();
}
}
You can use PLinq:
string firstResponse = urls
.AsParallel()
.Select(url => GetTimeSlot(url))
.FirstOrDefault(r => ! string.IsNullOrEmpty(r))
;
Related
Inside a loop I am creating a task using Task.Run(//...). Each task contains a WebRequest. At Task.WaitAll Result of one task getting overridden by Result of another Task. Whats wrong I am doing ?
When tried with debugger it works fine. Is it because of concurrency ? How to resolve this ?
Below is my code snippet:
SomeMethod()
{
//someItemList.Count() == 5
int r = 0;
Task<MyModel>[] myTaskList= new Task<MyModel>[5];
foreach(var item in someItemList){
Task<MyModel> t = Task<MyModel>.Run(() => { return
SomeOperationWithWebRequest(item); });
myTaskList[r] = t;
r++;
}
Task.WaitAll(myTaskList); //myTaskList[0].Result...myTaskList[4].Result all are having same output.
}
MyModel SomeOperationwithWebRequest(Item){
string URL = "SomeURLFromItem";
string DATA = "DATAfromItem"
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(URL);
request.Method = "POST";
request.ContentType = "application/json";
request.ContentLength = DATA.Length;
using (Stream webStream = request.GetRequestStream())
using (StreamWriter requestWriter = new StreamWriter(webStream, System.Text.Encoding.ASCII))
{
requestWriter.Write(DATA);
}
try
{
WebResponse webResponse = request.GetResponseAsync();
using (Stream webStream = webResponse.GetResponseStream() ?? Stream.Null)
using (StreamReader responseReader = new StreamReader(webStream))
{
//response
}
catch (Exception ex)
{
}
return new MyModel() { // properties
};
}
I think it works in debugging because you dont await the async WebRequest. Try this:
private readonly List<string> _someItemList = new List<string> { "t1", "t2", "t3" };
private async Task SomeMethodAsync()
{
var myTaskList = new List<Task<MyModel>>();
int counter = 0;
foreach (var item in _someItemList)
{
var t = Task.Run(() => SomeOperationWithWebRequestAsync(counter++));
myTaskList.Add(t);
}
await Task.WhenAll(myTaskList);
}
public async Task<MyModel> SomeOperationWithWebRequestAsync(int counter)
{
//do your async request, for simplicity I just do a delay
await Task.Delay(counter * 1000);
return new MyModel {Counter = counter };
}
public class MyModel
{
public int Counter { get; set; }
}
and use it like:
await SomeMethodAsync();
Also notice Task.WhenAll vs Task.WaitAll see e.g
WaitAll vs WhenAll
After reading other answers I can't realize why SendAsync is so slow.
Calling same endpoint from Postman, I got a response in 160ms.
Calling from the code below, takes 10 seconds. I'm using a c# desktop application to make the call.
public static async Task<string> GetToken()
{
var url = "....";
var dict = new Dictionary<string, string>();
dict.Add("username", "foo");
dict.Add("password", "bar");
using (var client = new HttpClient(
new HttpClientHandler
{
Proxy = null,
UseProxy = false
}))
{
//bypass SSL
ServicePointManager.ServerCertificateValidationCallback = new
RemoteCertificateValidationCallback
(
delegate { return true; }
);
var req = new HttpRequestMessage(HttpMethod.Post, url) { Content = new FormUrlEncodedContent(dict) };
var res = await client.SendAsync(req); //10 seconds here!
if (res.StatusCode != HttpStatusCode.OK)
return string.Empty;
var token = await JsonConvert.DeserializeObject<TokenResponse>(res.Content.ReadAsStringAsync());
return token.access_token;
}
}
Your code is tangled and ignores IDisposable and this: HttpClient is intended to be instantiated once per application, rather than per-use.
Make reusable method for other-type requests
private static readonly HttpClient client = new HttpClient();
private async Task<T> PostDataAsync<T>(string url, Dictionary<string, string> formData)
{
using (HttpContent content = new FormUrlEncodedContent(formData))
using (HttpResponseMessage response = await client.PostAsync(url, content).ConfigureAwait(false))
{
response.EnsureSuccessStatusCode(); // throws if 404, 500, etc.
string responseText = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
return JsonConvert.DeserializeObject<T>(responseText);
}
}
Usage
public static async Task<string> GetToken()
{
var url = "....";
var dict = new Dictionary<string, string>();
dict.Add("username", "foo");
dict.Add("password", "bar");
try
{
TokenResponse token = await PostDataAsync<TokenResponse>(url, dict);
return token.access_token;
}
catch (HttpRequestException ex)
{
// handle Exception here
return string.Empty;
}
}
I need to download numerous web pages' sources. So I need to do that as fast as possible. Here is my codes.
private static async Task<string> downloadsource(string link)
{
ServicePointManager.Expect100Continue = false;
WebRequest req = WebRequest.Create(link);
req.Proxy = null;
req.Method = "GET";
WebResponse res = await siteyeBaglantiTalebi.GetResponseAsync();
StreamReader read = new StreamReader(res.GetResponseStream());
return read.ReadToEnd();
}
List<string> links = new List<string>(){... including some web page links};
private static List<string> source_list(List<string> links)
{
List<string> sources = new List<string>();
for (int i = 0; i < links.Count; i++)
{
Task<string> _task = downloadsource(links[i]);
Console.WriteLine("Downloaded : " + i);
sources.Add(_task.Result);
}
return sources;
}
I was wondering if this code is the fastest way or it can be enhanced.
Can u pls help me with that ?
You are performing a _task.Result call inside each loop. Your code will run as fast as if you downloaded each page one after another if you code it like that.
Try this instead:
private async static Task<List<string>> source_list(List<string> links)
{
List<Task<string>> sources = new List<Task<string>>();
for (int i = 0; i < links.Count; i++)
{
Task<string> _task = downloadsource(links[i]);
Console.WriteLine("Downloading : " + i);
sources.Add(_task);
}
return (await Task.WhenAll(sources)).ToList();
}
This would be even better:
private async static Task<string[]> source_list(List<string> links)
{
return await Task.WhenAll(links.Select(l => downloadsource(l)));
}
Also I cleaned up your downloadsource method:
private static async Task<string> downloadsource(string link)
{
ServicePointManager.Expect100Continue = false;
WebRequest req = WebRequest.Create(link);
req.Proxy = null;
req.Method = "GET";
using (WebResponse res = await req.GetResponseAsync())
{
using (StreamReader read = new StreamReader(res.GetResponseStream()))
{
return read.ReadToEnd();
}
}
}
I am trying to write the following code using HttpClient and async, but it's not able return data.
WebRequest request = WebRequest.Create("some_url");
request.Headers.Add("cookie", "some_cookie");
Stream objStream = request.GetResponse().GetResponseStream();
StreamReader objReader = new StreamReader(objStream);
string sLine = "";
int i = 0;
while (sLine != null)
{
i++;
sLine = objReader.ReadLine();
if (sLine != null)
Console.WriteLine(sLine);
}
Here is what I tried.
using (var client = new HttpClient())
{
client.DefaultRequestHeaders.Add("cookie", "some_cookie");
using (var response = await client.GetAsync("some_url"))
{
string responseData = await response.Content.ReadAsStringAsync();
Console.WriteLine(responseData);
}
}
Any help will be appreciated.
Edit: Here is the code that works for me using HttpClient.
var baseAddress = new Uri(baseUrl);
using (var handler = new HttpClientHandler { UseCookies = false })
using (var client = new HttpClient(handler) { BaseAddress = baseAddress })
{
var requestMessage = new HttpRequestMessage(HttpMethod.Get, queryString);
requestMessage.Headers.Add("cookie", cookie);
var response = client.SendAsync(requestMessage);
response.Wait();
var content = response.Result.Content.ReadAsStringAsync();
content.Wait();
Console.WriteLine(content.Result);
}
Thanks for all the help.
You are doing the async and await, but you are not waiting for response data to be returned or reached so the following snippet will do the job :
Try this code :
static async Task<string> HttpGetResponse()
{
WebRequest request = WebRequest.Create("some_url");
request.Headers.Add("cookie", "some_cookie");
string responseData;
Stream objStream = request.GetResponse().GetResponseStream();
StreamReader objReader = new StreamReader(objStream);
string sLine = "";
int i = 0;
while (sLine != null)
{
i++;
sLine = objReader.ReadLine();
if (sLine != null)
Console.WriteLine(sLine);
}
using (var client = new HttpClient())
{
client.DefaultRequestHeaders.Add("cookie", "some_cookie");
using (var response = await client.GetAsync("some_url"))
{
responseData = await response.Content.ReadAsStringAsync();
Console.WriteLine(responseData);
}
}
return responseData;
}
in main call it like this :
static void Main(string[] args)
{
Task<string> t = HttpGetResponse();
//Do alot of work
t.Wait();
string response = t.Result;
Console.WriteLine(response);
}
Hope this was useful.
I am having a hard time convert the below code which i have created in 4.0 to 4.5 using HttpClient.
According to my understand i guess if i create multiple web requests in the GUI thread itself without blocking the GUI if i got with asynchronous requeest.
how to convert the below code to Asynchronous using HttpClient in 4.5
// This is what called when button is clicked
Task t3 = new Task(SpawnTask);
t3.Start();
//if noofthreads are less 50 then GUI is woking fine.. if number increases then takes much time for repaint..
//where as other softwares are working without any problem even if the threads are more than 500!! in the same system
public void SpawnTask()
{
try
{
ParallelOptions po = new ParallelOptions();
po.CancellationToken = cts.Token;
po.MaxDegreeOfParallelism = noofthreads;
Parallel.ForEach(
urls,
po,
url => checkpl(url));
}
catch (Exception ex)
{
}
}
public void checkpl(string url)
{
try
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
request.Timeout = 60*1000;
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
string stext = "";
using (BufferedStream buffer = new BufferedStream(response.GetResponseStream()))
{
using (StreamReader reader = new StreamReader(buffer))
{
stext = reader.ReadToEnd();
}
}
response.Close();
if (stext .IndexOf("domainname.com") != -1)
{
tfound = tfound + 1;
string lext = "Total Found : "+tfound.ToString();
label3.BeginInvoke(new InvokeDelegate(UpdateLabel), ltext);
slist.Add(url);
textBox2.BeginInvoke(new InvokeDelegate4(UpdateText), "Working Url " + url);
}
}
catch (Exception ex)
{
}
}
Since you are using .NET 4.5 you can use the new async and await keywords. Here is what it might look like.
private async void YourButton_Click(object sender, EventArgs args)
{
YourButton.Enabled = false;
try
{
var tasks = new List<Task>();
foreach (string url in Urls)
{
tasks.Add(CheckAsync(url));
}
await TaskEx.WhenAll(tasks);
}
finally
{
YourButton.Enabled = true;
}
}
private async Task CheckAsync(string url)
{
bool found = await UrlResponseContainsAsync(url, "domainname.com");
if (found)
{
slist.Add(url);
label3.Text = "Total Found: " + slist.Count.ToString();
textbox2.Text = "Working Url " + url;
}
}
private async Task<bool> UrlResponseContainsAsync(string url, string find)
{
var request = WebRequest.Create(url);
request.Timeout = 60 * 1000;
using (WebResponse response = await request.GetResponseAsync())
{
using (var buffer = new BufferedStream(response.GetResponseStream()))
using (var reader = new StreamReader(buffer))
{
string text = reader.ReadToEnd();
return text.Contains(find);
}
}
}