I am having a hard time convert the below code which i have created in 4.0 to 4.5 using HttpClient.
According to my understand i guess if i create multiple web requests in the GUI thread itself without blocking the GUI if i got with asynchronous requeest.
how to convert the below code to Asynchronous using HttpClient in 4.5
// This is what called when button is clicked
Task t3 = new Task(SpawnTask);
t3.Start();
//if noofthreads are less 50 then GUI is woking fine.. if number increases then takes much time for repaint..
//where as other softwares are working without any problem even if the threads are more than 500!! in the same system
public void SpawnTask()
{
try
{
ParallelOptions po = new ParallelOptions();
po.CancellationToken = cts.Token;
po.MaxDegreeOfParallelism = noofthreads;
Parallel.ForEach(
urls,
po,
url => checkpl(url));
}
catch (Exception ex)
{
}
}
public void checkpl(string url)
{
try
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
request.Timeout = 60*1000;
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
string stext = "";
using (BufferedStream buffer = new BufferedStream(response.GetResponseStream()))
{
using (StreamReader reader = new StreamReader(buffer))
{
stext = reader.ReadToEnd();
}
}
response.Close();
if (stext .IndexOf("domainname.com") != -1)
{
tfound = tfound + 1;
string lext = "Total Found : "+tfound.ToString();
label3.BeginInvoke(new InvokeDelegate(UpdateLabel), ltext);
slist.Add(url);
textBox2.BeginInvoke(new InvokeDelegate4(UpdateText), "Working Url " + url);
}
}
catch (Exception ex)
{
}
}
Since you are using .NET 4.5 you can use the new async and await keywords. Here is what it might look like.
private async void YourButton_Click(object sender, EventArgs args)
{
YourButton.Enabled = false;
try
{
var tasks = new List<Task>();
foreach (string url in Urls)
{
tasks.Add(CheckAsync(url));
}
await TaskEx.WhenAll(tasks);
}
finally
{
YourButton.Enabled = true;
}
}
private async Task CheckAsync(string url)
{
bool found = await UrlResponseContainsAsync(url, "domainname.com");
if (found)
{
slist.Add(url);
label3.Text = "Total Found: " + slist.Count.ToString();
textbox2.Text = "Working Url " + url;
}
}
private async Task<bool> UrlResponseContainsAsync(string url, string find)
{
var request = WebRequest.Create(url);
request.Timeout = 60 * 1000;
using (WebResponse response = await request.GetResponseAsync())
{
using (var buffer = new BufferedStream(response.GetResponseStream()))
using (var reader = new StreamReader(buffer))
{
string text = reader.ReadToEnd();
return text.Contains(find);
}
}
}
Related
I need to understand how SemaphoreSlim works with ThreadPool.
I need to read a file from the local disk of a computer, read each line, and then send that line as a HttpWebRequest to 2 different servers, and correspondingly get 2 HttpWebRequest back.
So lets say that file has 100 requests (this number can be in thousands or even more in a real-time scenario), and when all these requests will be sent I should get back 200 responses (like I mentioned each request should go to 2 different servers and fetch 2 responses from them). Here is my code:
static void Main(string[] args)
{
try
{
SendEntriesInFile(someFileOnTheLocaldisk);
Console.WriteLine();
}
catch (Exception e)
{
Console.WriteLine("Regression Tool Error: Major Unspecified Error:\n" + e);
}
}
}
public class MyParameters
{
}
private void SendEntriesInFile(FileInfo file)
{
static SemaphoreSlim threadSemaphore = new SemaphoreSlim(5, 10);
using (StreamReader reader = file.OpenText())
{
string entry = reader.ReadLine();
while (!String.IsNullOrEmpty(entry))
{
MyParameters myParams = new MyParameters(entry, totalNumberOfEntries, serverAddresses, requestType, fileName);
threadSemaphore.Wait();
ThreadPool.QueueUserWorkItem(new WaitCallback(Send), requestParams);
entry = reader.ReadLine();
}
}
}
private void Send(object MyParameters)
{
MyParameters myParams = (MyParameters)MyParameters;
for(int i=0; i < myParams.ServerAddresses.Count; i++)
{
byte[] bytesArray = null;
bytesArray = Encoding.UTF8.GetBytes(myParams.Request);
HttpWebRequest webRequest = null;
if (reqParams.TypeOfRequest == "tlc")
{
webRequest = (HttpWebRequest)WebRequest.Create(string.Format("http://{0}:{1}{2}", myParams .ServerAddresses[i], port, "/SomeMethod1"));
}
else
{
webRequest = (HttpWebRequest)WebRequest.Create(string.Format("http://{0}:{1}{2}", myParams .ServerAddresses[i], port, "/SomeMethod2"));
}
if (webRequest != null)
{
webRequest.Method = "POST";
webRequest.ContentType = "application/x-www-form-urlencoded";
webRequest.ContentLength = bytesArray.Length;
webRequest.Timeout = responseTimeout;
webRequest.ReadWriteTimeout = transmissionTimeout;
webRequest.ServicePoint.ConnectionLimit = maxConnections;
webRequest.ServicePoint.ConnectionLeaseTimeout = connectionLifeDuration;
webRequest.ServicePoint.MaxIdleTime = maxConnectionIdleTimeout;
webRequest.ServicePoint.UseNagleAlgorithm = nagleAlgorithm;
webRequest.ServicePoint.Expect100Continue = oneHundredContinue;
using (Stream requestStream = webRequest.GetRequestStream())
{
//Write the request through the request stream
requestStream.Write(bytesArray, 0, bytesArray.Length);
requestStream.Flush();
}
string response = "";
using (HttpWebResponse httpWebResponse = (HttpWebResponse)webRequest.GetResponse())
{
if (httpWebResponse != null)
{
using (Stream responseStream = httpWebResponse.GetResponseStream())
{
using (StreamReader stmReader = new StreamReader(responseStream))
{
response = stmReader.ReadToEnd();
string fileName = "";
if(i ==0)
{
fileName = Name is generated through some logic here;
}
else
{
fileName = Name is generated through some logic here;
}
using (StreamWriter writer = new StreamWriter(fileName))
{
writer.WriteLine(response);
}
}
}
}
}
}
}
Console.WriteLine(" Release semaphore: ---- " + threadSemaphore.Release());
}
The only thing is that I'm confused with that when I do something like above my Semaphore allow 5 threads to concurrently execute the Send() method and 5 other threads they wait in a queue for their turn. Since my file contains 100 requests so I should get back 200 responses. Every time I ended up getting only 109 or 107 or 108 responses back. Why I don't get all the 200 responses back. Though using a different code (not discussed here) when I send lets say 10 requests in parallel on 10 different created threads (not using the ThreadPool rather creating threads on-demand) I get back all the 200 responses.
Lots of very helpful comments. To add to them, I would recommend using just pure async io for this.
I would recommend applying the Semaphore as a DelegatingHandler.
You want to register this class.
public class LimitedConcurrentHttpHandler
: DelegatingHandler
{
private readonly SemaphoreSlim _concurrencyLimit = new(8);
public LimitedConcurrentHttpHandler(){ }
public LimitedConcurrentHttpHandler(HttpMessageHandler inner) : base(inner) {}
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
await _concurrencyLimit.Wait(cancellationToken);
try
{
return await base.SendAsync(request, cancellationToken);
}
finally
{
_concurrencyLimit.Release();
}
}
}
This way, you can concentrate on your actual business logic within your actual code.
Inside a loop I am creating a task using Task.Run(//...). Each task contains a WebRequest. At Task.WaitAll Result of one task getting overridden by Result of another Task. Whats wrong I am doing ?
When tried with debugger it works fine. Is it because of concurrency ? How to resolve this ?
Below is my code snippet:
SomeMethod()
{
//someItemList.Count() == 5
int r = 0;
Task<MyModel>[] myTaskList= new Task<MyModel>[5];
foreach(var item in someItemList){
Task<MyModel> t = Task<MyModel>.Run(() => { return
SomeOperationWithWebRequest(item); });
myTaskList[r] = t;
r++;
}
Task.WaitAll(myTaskList); //myTaskList[0].Result...myTaskList[4].Result all are having same output.
}
MyModel SomeOperationwithWebRequest(Item){
string URL = "SomeURLFromItem";
string DATA = "DATAfromItem"
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(URL);
request.Method = "POST";
request.ContentType = "application/json";
request.ContentLength = DATA.Length;
using (Stream webStream = request.GetRequestStream())
using (StreamWriter requestWriter = new StreamWriter(webStream, System.Text.Encoding.ASCII))
{
requestWriter.Write(DATA);
}
try
{
WebResponse webResponse = request.GetResponseAsync();
using (Stream webStream = webResponse.GetResponseStream() ?? Stream.Null)
using (StreamReader responseReader = new StreamReader(webStream))
{
//response
}
catch (Exception ex)
{
}
return new MyModel() { // properties
};
}
I think it works in debugging because you dont await the async WebRequest. Try this:
private readonly List<string> _someItemList = new List<string> { "t1", "t2", "t3" };
private async Task SomeMethodAsync()
{
var myTaskList = new List<Task<MyModel>>();
int counter = 0;
foreach (var item in _someItemList)
{
var t = Task.Run(() => SomeOperationWithWebRequestAsync(counter++));
myTaskList.Add(t);
}
await Task.WhenAll(myTaskList);
}
public async Task<MyModel> SomeOperationWithWebRequestAsync(int counter)
{
//do your async request, for simplicity I just do a delay
await Task.Delay(counter * 1000);
return new MyModel {Counter = counter };
}
public class MyModel
{
public int Counter { get; set; }
}
and use it like:
await SomeMethodAsync();
Also notice Task.WhenAll vs Task.WaitAll see e.g
WaitAll vs WhenAll
I need to download numerous web pages' sources. So I need to do that as fast as possible. Here is my codes.
private static async Task<string> downloadsource(string link)
{
ServicePointManager.Expect100Continue = false;
WebRequest req = WebRequest.Create(link);
req.Proxy = null;
req.Method = "GET";
WebResponse res = await siteyeBaglantiTalebi.GetResponseAsync();
StreamReader read = new StreamReader(res.GetResponseStream());
return read.ReadToEnd();
}
List<string> links = new List<string>(){... including some web page links};
private static List<string> source_list(List<string> links)
{
List<string> sources = new List<string>();
for (int i = 0; i < links.Count; i++)
{
Task<string> _task = downloadsource(links[i]);
Console.WriteLine("Downloaded : " + i);
sources.Add(_task.Result);
}
return sources;
}
I was wondering if this code is the fastest way or it can be enhanced.
Can u pls help me with that ?
You are performing a _task.Result call inside each loop. Your code will run as fast as if you downloaded each page one after another if you code it like that.
Try this instead:
private async static Task<List<string>> source_list(List<string> links)
{
List<Task<string>> sources = new List<Task<string>>();
for (int i = 0; i < links.Count; i++)
{
Task<string> _task = downloadsource(links[i]);
Console.WriteLine("Downloading : " + i);
sources.Add(_task);
}
return (await Task.WhenAll(sources)).ToList();
}
This would be even better:
private async static Task<string[]> source_list(List<string> links)
{
return await Task.WhenAll(links.Select(l => downloadsource(l)));
}
Also I cleaned up your downloadsource method:
private static async Task<string> downloadsource(string link)
{
ServicePointManager.Expect100Continue = false;
WebRequest req = WebRequest.Create(link);
req.Proxy = null;
req.Method = "GET";
using (WebResponse res = await req.GetResponseAsync())
{
using (StreamReader read = new StreamReader(res.GetResponseStream()))
{
return read.ReadToEnd();
}
}
}
I have 5 URL and I want to make a Http request for each one, And waiting for the first response that has the conditions.
List<string> urls; // url1, url2, ......
ParallelLoopResult result = Parallel.ForEach(urls, url=> GetTimeSlot(url));
private string GetTimeSlot(string url)
{
HttpWebRequest wr = (HttpWebRequest)WebRequest.Create(url);
HttpWebResponse response = (HttpWebResponse)wr.GetResponse();
string responseString = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding(response.CharacterSet)).ReadToEnd();
if (responseString.Length < 6)
return ""; //PARALEL RESUME
else
return responseString; //PARALEL ENDS
}
I need only the first response. Is it possible with Parallel or is there any better way? Thanks.
Parallel.ForEach will be fine to use especially for your use case.Just use a cancellation token to stop all other running tasks.
static void Main(string[] args)
{
var cts = new CancellationTokenSource();
var _lock = new Object();
var po = new ParallelOptions();
po.CancellationToken = cts.Token;
po.MaxDegreeOfParallelism = System.Environment.ProcessorCount;
var listOfUrls = new List<string>() { "url1", "url2" };
var responsResult = "";
try
{
Parallel.ForEach(listOfUrls, po, (url) =>
{
po.CancellationToken.ThrowIfCancellationRequested();
HttpWebRequest wr = (HttpWebRequest)WebRequest.Create(url);
HttpWebResponse response = (HttpWebResponse)wr.GetResponse();
string responseString = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding(response.CharacterSet)).ReadToEnd();
lock (_lock)
{
if (responseString.Length > 6)
{
responsResult = responseString;
cts.Cancel();
}
}
});
}
catch (OperationCanceledException e)
{
//cancellation was requested
}
finally
{
cts.Dispose();
}
}
You can use PLinq:
string firstResponse = urls
.AsParallel()
.Select(url => GetTimeSlot(url))
.FirstOrDefault(r => ! string.IsNullOrEmpty(r))
;
Sorry for my english. I have a method in which I send the StorageFile to the server. I tried using Windows.Web.Http.HttpClient, but does not work ( getting an invalid response from the server ) , so I use System.Net.Http.HttpClient.
Here is my code :
public static async void Upload(string uri, StorageFile data, Action<double> progressCallback = null)
{
try
{
byte[] fileBytes =await ReadFile(data);
System.Net.Http.HttpClient client = new System.Net.Http.HttpClient();
MultipartContent content = new System.Net.Http.MultipartFormDataContent();
var file1 = new ByteArrayContent(fileBytes);
file1.Headers.ContentDisposition = new ContentDispositionHeaderValue("form-data")
{
Name = "file1",
FileName = data.Name,
};
content.Add(file1);
System.Net.Http.HttpResponseMessage response = await client.PostAsync(uri, content);
response.EnsureSuccessStatusCode();
var raw_response = await response.Content.ReadAsByteArrayAsync();
var r2 = Encoding.UTF8.GetString(raw_response, 0, raw_response.Length);
if (r2[0] == '\uFEFF')
{
r2 = r2.Substring(1);
}
Logger.Info(r2);
}
catch (Exception exc)
{
Logger.Error( exc);
}
}
Whether it is possible to change the code to receive progress about downloading a file in callback function?
On Windows Runtime you can try to switch to Windows.Web.HttpClient class. Its PostAsync method returns IAsyncOperationWithProgress<HttpResponseMessage, HttpProgress> interface. This interface has Progress event to which you can simply subscribe before awaiting the result.
Simplest way to upload file with progress
I had the same issue, and after some tries found out that you can easily get byte-accurate progress by tracking the Position of the FileStream of the file that you are going to upload.
This is a sample code that shows that.
FileStream fileToUpload = File.OpenRead(#"C:\test.mp3");
HttpContent content = new StreamContent(fileToUpload);
HttpRequestMessage msg = new HttpRequestMessage{
Content=content,
RequestUri = new Uri(--yourUploadURL--)
}
bool keepTracking = true; //to start and stop the tracking thread
new Task(new Action(() => { progressTracker(fileToUpload, ref keepTracking); })).Start();
var result = httpClient.SendAsync(msg).Result;
keepTracking = false; //stops the tracking thread
And define progressTracker() as
void progressTracker(FileStream streamToTrack, ref bool keepTracking)
{
int prevPos = -1;
while (keepTracking)
{
int pos = (int)Math.Round(100 * (streamToTrack.Position / (double)streamToTrack.Length));
if (pos != prevPos)
{
Console.WriteLine(pos + "%");
}
prevPos = pos;
Thread.Sleep(100); //only update progress every 100ms
}
}
And this solved my problem.