We Keep Coding

C# UWP Login website using cookies

I'm trying to convert a WPF application to UWP but am struggeling on how to get the conenction to work.
The WPF application connects to a website, get cookies, read a token from html and then post data to login.
In WPF I had this class:
// CookieAwareWebClient.cs
public class CookieAwareWebClient : WebClient
{
public string Method;
public CookieContainer CookieContainer { get; set; }
public CookieAwareWebClient()
: this(new CookieContainer())
{
}
public CookieAwareWebClient(CookieContainer cookies)
{
this.CookieContainer = cookies;
}
protected override WebRequest GetWebRequest(Uri address)
{
WebRequest request = base.GetWebRequest(address);
if (request is HttpWebRequest)
{
(request as HttpWebRequest).CookieContainer = this.CookieContainer;
(request as HttpWebRequest).ServicePoint.Expect100Continue = false;
(request as HttpWebRequest).UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:18.0) Gecko/20100101 Firefox/18.0";
(request as HttpWebRequest).Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
(request as HttpWebRequest).KeepAlive = true;
(request as HttpWebRequest).AutomaticDecompression = DecompressionMethods.Deflate |
DecompressionMethods.GZip;
if (Method == "POST")
{
(request as HttpWebRequest).ContentType = "application/x-www-form-urlencoded";
}
}
HttpWebRequest httpRequest = (HttpWebRequest)request;
httpRequest.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
return httpRequest;
}
protected override WebResponse GetWebResponse(WebRequest request)
{
WebResponse response = base.GetWebResponse(request);
String setCookieHeader = response.Headers[HttpResponseHeader.SetCookie];
if (setCookieHeader != null)
{
try
{
if (setCookieHeader != null)
{
Cookie cookie = new Cookie(); //create cookie
this.CookieContainer.Add(cookie);
}
}
catch (Exception)
{
}
}
return response;
}
}
,that is used like this:
// Program.cs
// ...
CookieAwareWebClient client = new CookieAwareWebClient(cookieJar);
string response = client.DownloadString("LINK TO WEBSITE TO GET COOKIES");
// Get the csrf token from response html
var tokenValue = Regex.Match(response, "name='csrf_token'
value='(.+?)'").Groups[1].Value;
// Prepare values to send
string token = $"csrf_token={tokenValue}&";
string email = $"email={email}&";
string password = $"password={password}&";
string rememberme = $"rememberme=on&";
string submit = $"submit=Login";
string postData = token + email + password + rememberme + submit;
client.Method = "POST";
response = client.UploadString("LINK TO LOGIN ACTION", postData);
This works flawless in WPF. How can I achieve someting like this in UWP?
I tried using HttpClient, but I either get bad response, javascript doesnt load, or I can't read the response.
So far, I tried to just connect to the normal site and read the html, to get the token. But I get a response error.
var handler = new HttpClientHandler { AllowAutoRedirect = true };
var client = new HttpClient(handler);
client.DefaultRequestHeaders.TryAddWithoutValidation("UserAgent", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:18.0) Gecko/20100101 Firefox/18.0");
client.DefaultRequestHeaders.TryAddWithoutValidation("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
client.DefaultRequestHeaders.TryAddWithoutValidation("KeepAlive", "true");
client.DefaultRequestHeaders.TryAddWithoutValidation("ServicePoint.Expect100Continue", "false");
var response = await client.GetAsync(new Uri("LINK"));
// ERROR => System.Net.Http.HttpRequestException: "Response status code does not indicate success: 503 ()."
response.EnsureSuccessStatusCode();
var html = await response.Content.ReadAsStringAsync();

You need to add a CookieContainer to the handler in order to be able to access the cookies return from the request
//have a cookie container before making the request
var cookies = new CookieContainer();
var handler = new HttpClientHandler() {
AllowAutoRedirect = true,
CookieContainer = cookies,
AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip,
};
//Handle TLS protocols (https)
System.Net.ServicePointManager.SecurityProtocol =
System.Net.SecurityProtocolType.Tls
| System.Net.SecurityProtocolType.Tls11
| System.Net.SecurityProtocolType.Tls12;
var client = new HttpClient(handler);
client.DefaultRequestHeaders.TryAddWithoutValidation("UserAgent", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:18.0) Gecko/20100101 Firefox/18.0");
client.DefaultRequestHeaders.TryAddWithoutValidation("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
client.DefaultRequestHeaders.TryAddWithoutValidation("KeepAlive", "true");
client.DefaultRequestHeaders.TryAddWithoutValidation("ServicePoint.Expect100Continue", "false");
var uri = new Uri("Link");
var response = await client.GetAsync(uri);
var html = await response.Content.ReadAsStringAsync();
// Get the csrf token from response html
var tokenValue = Regex.Match(html, "name='csrf_token' value='(.+?)'").Groups[1].Value;
//...Prepare values to send

The same code works using HttpWebRequest but not using HttpRequestMessage

I've created HttpClient that I'm using for sending requests:
public static void Initialize()
{
handler = new HttpClientHandler() { UseCookies = false, AllowAutoRedirect = true };
http = new HttpClient(handler) { BaseAddress = new Uri("http://csgolounge.com/mytrades") };
http.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36");
}
After that I'm creating instance of custom class that stores the cookies string for an account (something like id=xxxxxxxx; tkz=xxxxxxxxxx; token=xxxxxxxxxxx.
That's how I'm sending a post request:
public async Task Bump()
{
//if (Bumpable)
//{
var req = new HttpRequestMessage(HttpMethod.Post, "http://www.csgolounge.com/ajax/bumpTrade.php");
req.Headers.Add("Cookie", cookieString);
req.Headers.Add("X-Requested-With", "XMLHttpRequest");
req.Headers.Add("Referer", "http://csgolounge.com/mytrades"); //Not really sure if this does anything but I've run out of smart ideas long time ago
/*Dictionary<string, string> postData = new Dictionary<string, string>()
{
{"trade", offer_id}
};
var encoded = new FormUrlEncodedContent(postData);
*/
req.Content = new StringContent("&trade="+Offer_id, Encoding.UTF8, "application/x-www-form-urlencoded"); //Desperation.. decided to change the encoded dictionary to StringContent
var res = await SteamAccount.http.SendAsync(req);
var html = await res.Content.ReadAsStringAsync();
//}
}
I don't get what's wrong with this code. It seems correct to me.
Also, when I set AllowAutoRedirect = false it returns 301: Moved Permanently error, while normally it returns 200 with no HTML no matter what I pass as content.
What am I doing wrong?
Edit: Here's the JavaScript function I'm basing my request on:
function bumpTrade(trade) {
$.ajax({
type: "POST",
url: "ajax/bumpTrade.php",
data: "trade=" + trade
});
}
I've worked with more complex AJAX before, but this just doesn't seem to work no matter what I do.
Edit: I've lost my patience and switched to HttpWebRequest instead.
Now the method looks like this:
public async Task BumpLegacy()
{
while (true)
{
try
{
HttpWebRequest httpWebRequest = (HttpWebRequest)WebRequest.Create("http://csgolounge.com/ajax/bumpTrade.php");
var cc = new CookieContainer();
MatchCollection mc = Regex.Matches(Account.CookieString, #"\s?([^=]+)=([^;]+);");
foreach (Match m in mc)
cc.Add(new Cookie(m.Groups[1].Value, m.Groups[2].Value, "/", "csgolounge.com"));
httpWebRequest.CookieContainer = cc;
byte[] bytes = Encoding.ASCII.GetBytes("trade=" + Offer_id);
httpWebRequest.Referer = "http://csgolounge.com/mytrades";
httpWebRequest.Headers.Add("X-Requested-With", "XMLHttpRequest");
httpWebRequest.Method = "POST";
httpWebRequest.ContentType = "application/x-www-form-urlencoded; charset=UTF-8";
httpWebRequest.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36";
httpWebRequest.ContentLength = (long)bytes.Length;
var g = await httpWebRequest.GetRequestStreamAsync();
await g.WriteAsync(bytes, 0, bytes.Count());
g.Close();
var res = await httpWebRequest.GetResponseAsync();
res.Close();
break;
}
catch
{
}
}
}
Maybe I'm just dumb but for me it doesn't seem all that different. Are there some key differences that can be the cause?

Here is code from one of my working systems that submits a POST request through an HTTPClient.
[Route("resource")]
public async Task<dynamic> CreateResource([FromBody]Resource resource)
{
if (resource == null) return BadRequest();
dynamic response = null;
resource.Topic = GetDataFromSomewhereElse();
var message = new PostMessage(resource).BodyContent;
dynamic postRequest = new
{
Message = message
};
var post = JsonConvert.SerializeObject(postRequest);
HttpContent content = new StringContent(post, Encoding.UTF8, "application/json");
using (var client = new HttpClient())
{
client.Timeout = TimeSpan.FromMinutes(1);
client.DefaultRequestHeaders.Accept.Clear();
client.DefaultRequestHeaders.Accept.Add(new System.Net.Http.Headers.MediaTypeWithQualityHeaderValue("application/json"));
try
{
client.BaseAddress = #"http://localhost:51145/";
HttpResponseMessage postResponse = await client.PostAsync("Resource", content); //"Resource" is a route exposed on the remote host
string json = await postResponse.Content.ReadAsStringAsync();
if (postResponse.StatusCode == HttpStatusCode.BadRequest) return BadRequest();
if (postResponse.StatusCode == HttpStatusCode.InternalServerError) return InternalServerError();
if (postResponse.StatusCode == HttpStatusCode.NotFound) return NotFound();
return json;
}
catch(Exception ex)
{
return InternalServerError(ex);
}
}
}
[Edit]
The "PostMessage" was modified to remove domain-specific details. Here is how BodyContent is defined inside the real "PostMessage" from my solution, to provide you enough context to understand what that "message" actually is and how it works into the sample.
public string BodyContent
{
get
{
string content = "";
Type type = this.GetType();
Assembly assembly = Assembly.GetExecutingAssembly();
string resource = String.Format("{0}.{1}", type.Namespace, this.EmbeddedResourceName);
Stream stream = assembly.GetManifestResourceStream(resource);
StreamReader reader = new StreamReader(stream);
content = reader.ReadToEnd();
return content;
}
}
...and here is PostRequest (again, with domain-specific details trimmed)
public class PostRequest
{
public string Message { get;set; }
}

HttpWebrequest/WebClient/HttpClient/CookieContainer bug? Does not add cookie if Response Uri does not contain it's path

I was trying to log in to my WordPress admin panel using C# (WebClient/HttpWebRequest).
I send a POST request to /wp-login.php.
It responds with cookies like this:
Set-Cookie: wordpress_26...cf9e; path=/wordpress/wp-content/plugins; httponly
Set-Cookie: wordpress_26...cf9e; path=/wordpress/wp-admin; httponly
Set-Cookie: wordpress_logged_in_26...43b3; path=/wordpress/; httponly
And it also redirects to /wp-admin/: Location: http://109.120.169.99/wordpress/wp-admin/
The problem is that the second cookie (with path=/wp-admin) is not added to CookieContainer, so login fails.
I looked into HttpWebRequest source code and http://referencesource.microsoft.com/#System/net/System/Net/_CookieModule.cs (OnReceiveHeaders) and found out that it uses ResponseUri for all cookies.
I tried to add the cookie manually using response uri and got an exception that path is invalid. After that I found this answer CookieException with CookieContainer: The 'Path' part of the cookie is invalid : it says that the Uri has to match the path from the cookie when adding to CookieContainer.
I ended up disabling AllowAutoRedict (handling it manually) and manually adding these cookies.
Here is my code:
public static void Auth(string url, string username, string password)
{
var webClient = new CookieAwareWebClient();
// load page to get some cookies
string loginPageHtml = webClient.DownloadString(url);
webClient.Headers.Add(HttpRequestHeader.ContentType, "application/x-www-form-urlencoded");
webClient.Headers.Add(HttpRequestHeader.Referer, url);
var postValues = new NameValueCollection()
{
{"log", username},
{"pwd", password},
{"wp-sumbit", "Log In"},
{"redirect_to", url.Replace("wp-login.php", "wp-admin/") },
{"testcookie", "1"}
};
webClient.AllowRedirects = false; // to handle cookies and redirect manually
byte[] response = webClient.UploadValues(url, postValues);
string html = Encoding.UTF8.GetString(response);
// <FIX>, handle cookies and redirect manually
string cookies = webClient.ResponseHeaders[HttpResponseHeader.SetCookie];
var cookiesArr = cookies.Split(',');
foreach (var cookieStr in cookiesArr)
{
var parts = cookieStr.Split(';').Select(s => s.Trim());
foreach (var part in parts)
{
if (part.Contains("path="))
{
string path = part.Replace("path=", "");
var uri = new Uri(url);
if (path != "/" && !uri.LocalPath.Contains(path))
{
var uriBuilder = new UriBuilder(uri.Scheme, uri.Host, 80,
path);
webClient.CookieContainer.SetCookies(uriBuilder.Uri, cookieStr);
}
}
}
}
// </FIX>
while (webClient.StatusCode() == HttpStatusCode.Redirect)
{
string redirectUrl = webClient.ResponseHeaders[HttpResponseHeader.Location];
html = webClient.DownloadString(redirectUrl);
}
if (html.Contains("?action=logout"))
Console.WriteLine("Login success");
else
Console.WriteLine("Login fail");
}
WebClient:
// WebClient with CookieContainer
public class CookieAwareWebClient : WebClient
{
private WebRequest _request = null;
private WebResponse _response = null;
public CookieContainer CookieContainer { get; private set; }
public string UserAgent { get; set; }
public bool AllowRedirects { get; set; }
public CookieAwareWebClient()
{
UserAgent = "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.76 Safari/537.36";
CookieContainer = new CookieContainer();
AllowRedirects = true;
}
protected override WebRequest GetWebRequest(Uri address)
{
_request = base.GetWebRequest(address);
if (_request is HttpWebRequest)
{
var httpRequest = (HttpWebRequest)_request;
httpRequest.CookieContainer = CookieContainer;
httpRequest.UserAgent = UserAgent;
httpRequest.AllowAutoRedirect = AllowRedirects;
httpRequest.ProtocolVersion = HttpVersion.Version11;
httpRequest.Timeout = 50000;
}
return _request;
}
protected override WebResponse GetWebResponse(WebRequest request)
{
_response = base.GetWebResponse(request);
return _response;
}
// Returns status code of the last response
public HttpStatusCode StatusCode()
{
var httpResponse = _response as HttpWebResponse;
if (httpResponse == null)
throw new InvalidOperationException("Unable to retrieve the status code, maybe you have not made a request yet.");
var result = httpResponse.StatusCode;
return result;
}
}
I also tried using .NET 4.5 HttpClient but the same result (I think it uses HttpWebRequest underneath too):
public static void Auth2(string url, string username, string password)
{
using (var httpClient = new HttpClient())
{
var loginPageHtml = httpClient.GetStringAsync(url).Result;
var postContent = new List<KeyValuePair<string, string>>()
{
new KeyValuePair<string, string>("log", username),
new KeyValuePair<string, string>("pwd", password),
new KeyValuePair<string, string>("wp-submit", "Log In"),
new KeyValuePair<string, string>("redirect_to", url.Replace("wp-login.php", "wp-admin/")),
new KeyValuePair<string, string>("testcookie", "1")
};
var response = httpClient.PostAsync(url, new FormUrlEncodedContent(postContent)).Result;
string html = response.Content.ReadAsStringAsync().Result;
if (html.Contains("?action=logout"))
Console.WriteLine("Login success");
else
Console.WriteLine("Login fail");
}
}
Am I doing something wrong or is it a bug in HttpWebRequest/CookieContainer?
Here is the full source code for convenience if someone wants to test it: https://gist.github.com/AlexP11223/5c176972426605ee2112 (my test website and login/password also should work)
And Fiddler logs:
http://1drv.ms/1qyAgGi — webbrowser (Chrome), it adds this cookie
http://1drv.ms/1kqsLDI — WebClient
http://1drv.ms/1kqsC2Y —
HttpClient

HttpWebReponse Cookies Not Setting For Redirect

I'm trying to do the following:
send a GET request to fetch a login page (which prompts for username, password, and sets a cookie)
build a POST request that sends the cookie from #1 and a body of the username/password (this returns a Set-Cookie and redirects to the website's landing page for logged in users)
My trouble is with the 302 redirect. The webserver is returning a 302 with a Set-Cookie, but when the HttpWebRequests auto-redirects, it doesn't pass along the now updated cookie. To get around that, I'm trying to set .AllowAutoRedirect = false, saving the cookies in a CookieCollection, then building a 3rd HTTP request: a GET to the final 302 location. Unfortunately, I can't set the cookies on this request. I'm not sure why and it's driving me mad.
The HTTP requests are, in order, named request, postRequest, redirectRequest.
string loginGetUrl = "https://<..>/signin.htm";
string loginPostUrl = "https://<..>/j_acegi_security_check";
string loginRedirectUrl = "https://<..>/centraladmin/poslinks.htm";
string postData = String.Format("j_username={0}&j_password={1}", username, password);
CookieCollection cookies = new CookieCollection();
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(loginGetUrl);
request.CookieContainer = new CookieContainer();
request.CookieContainer.Add(cookies);
//Get the response from the server and save the cookies from the first request..
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
cookies = response.Cookies;
HttpWebRequest postRequest = (HttpWebRequest)WebRequest.Create(loginPostUrl);
postRequest.CookieContainer = new CookieContainer();
// Add the received Cookies from the HTTP Get
postRequest.CookieContainer.Add(cookies);
postRequest.Method = WebRequestMethods.Http.Post;
postRequest.UserAgent = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.121 Safari/535.2";
postRequest.AllowWriteStreamBuffering = false;
postRequest.ProtocolVersion = HttpVersion.Version11;
postRequest.AllowAutoRedirect = false;
postRequest.ContentType = "application/x-www-form-urlencoded";
byte[] byteArray = Encoding.ASCII.GetBytes(postData);
postRequest.ContentLength = byteArray.Length;
Stream newStream = postRequest.GetRequestStream(); //open connection
newStream.Write(byteArray, 0, byteArray.Length); // Send the data.
newStream.Close();
HttpWebResponse postResponse = (HttpWebResponse)postRequest.GetResponse();
// Save the cookies from the POST login request, then send them on to the redirected URL
cookies = postResponse.Cookies;
HttpWebRequest redirectRequest = (HttpWebRequest)WebRequest.Create(loginRedirectUrl);
redirectRequest.CookieContainer = new CookieContainer();
// add cookies from POST
redirectRequest.CookieContainer.Add(cookies);
HttpWebResponse redirectResponse = (HttpWebResponse)redirectRequest.GetResponse();
At redirectRequest.CookieContainer.Add(cookies);, the cookies object contains the correct cookie. But when I look with Fiddler, I only see this info:
GET https://<...>/centraladmin/poslinks.htm HTTP/1.1
Host: host:port
I'm kind of banging my head on the wall at this point. Any suggestions? Am I referencing something wrong? Beware, I don't usually write C# code

I wasn't able to resolve this on my own, but did find a useful code snippet from this blog post by #malte-clasen. The code is on Github and I've attached it here for retention.
I removed the async components as it wasn't necessary in my code.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;
namespace DracWake.Core
{
public class WebClient : IWebClient
{
private readonly CookieContainer _cookies = new CookieContainer();
private HttpWebRequest CreateRequest(Uri uri)
{
var request = HttpWebRequest.CreateHttp(uri);
request.AllowAutoRedirect = false;
request.CookieContainer = _cookies;
SetHeaders(request);
var defaultValidator = System.Net.ServicePointManager.ServerCertificateValidationCallback;
request.ServerCertificateValidationCallback =
(sender, certificate, chain, sslPolicyErrors) =>
certificate.Subject.Contains("O=DO_NOT_TRUST, OU=Created by http://www.fiddler2.com")
|| (certificate.Subject == "CN=DRAC5 default certificate, OU=Remote Access Group, O=Dell Inc., L=Round Rock, S=Texas, C=US")
|| (defaultValidator != null && defaultValidator(request, certificate, chain, sslPolicyErrors));
return request;
}
private async Task<string> DecodeResponse(HttpWebResponse response)
{
foreach (System.Net.Cookie cookie in response.Cookies)
{
_cookies.Add(new Uri(response.ResponseUri.GetLeftPart(UriPartial.Authority)), cookie);
}
if (response.StatusCode == HttpStatusCode.Redirect)
{
var location = response.Headers[HttpResponseHeader.Location];
if (!string.IsNullOrEmpty(location))
return await Get(new Uri(location));
}
var stream = response.GetResponseStream();
var buffer = new System.IO.MemoryStream();
var block = new byte[65536];
var blockLength = 0;
do{
blockLength = stream.Read(block, 0, block.Length);
buffer.Write(block, 0, blockLength);
}
while(blockLength == block.Length);
return Encoding.UTF8.GetString(buffer.GetBuffer());
}
public async Task<string> Get(Uri uri)
{
var request = CreateRequest(uri);
var response = (HttpWebResponse) await request.GetResponseAsync();
return await DecodeResponse(response);
}
private void SetHeaders(HttpWebRequest request)
{
request.Accept = "text/html, application/xhtml+xml, */*";
request.UserAgent = "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; WOW64; Trident/6.0)";
request.ContentType = "application/x-www-form-urlencoded";
request.Headers[HttpRequestHeader.AcceptLanguage] = "en-US,en;q=0.8,de-DE;q=0.5,de;q=0.3";
request.Headers[HttpRequestHeader.AcceptEncoding] = "gzip, deflate";
request.Headers[HttpRequestHeader.CacheControl] = "no-cache";
}
public async Task<string> Post(Uri uri, byte[] data)
{
var request = CreateRequest(uri);
request.Method = "POST";
request.GetRequestStream().Write(data, 0, data.Length);
var response = (HttpWebResponse) await request.GetResponseAsync();
return await DecodeResponse(response);
}
}
}
The DecodeResponse resolved my problem.

C# Error on multithreading webrequests

i am trying to make webrequests with multiple threads but if i try with more than 2 i get error
Index was outside the bonds of the array
on this line:
string username = ScrapeBox1.Lines[NamesCounter].ToString();
Here's the code:
while (working)
{
while (usernamescount > NamesCounter)
{
string username = ScrapeBox1.Lines[NamesCounter].ToString();
string url = "http://www.someforum.com/members/" + username + ".html";
var request = (HttpWebRequest)(WebRequest.Create(url));
var response = request.GetResponse();
request.UserAgent = "Mozilla/5.0 (Windows NT 6.1; rv:16.0) Gecko/20100101 Firefox/16.0";
using (var responseStream = response.GetResponseStream())
{
using (var responseStreamReader = new StreamReader(responseStream))
{
var serverResponse = responseStreamReader.ReadToEnd();
int startpoint = serverResponse.IndexOf("Contact Info</span>");
try
{
string strippedResponse = serverResponse.Remove(0, startpoint);
ExtractEmails(strippedResponse);
}
catch { }
}
}
NamesCounter++;
textBox1.Text = NamesCounter.ToString();
}
}

This code is not thread safe.
You need the code for performing an HttpWebRequest to be atomic and outside the context of looping through the collection.
For example
public void MakeHttpWebRequest(string userName)
{
string url = "http://www.someforum.com/members/" + userName + ".html";
var request = (HttpWebRequest)(WebRequest.Create(url));
var response = request.GetResponse();
request.UserAgent = "Mozilla/5.0 (Windows NT 6.1; rv:16.0) Gecko/20100101 Firefox/16.0";
using (var responseStream = response.GetResponseStream())
{
using (var responseStreamReader = new StreamReader(responseStream))
{
var serverResponse = responseStreamReader.ReadToEnd();
int startpoint = serverResponse.IndexOf("Contact Info</span>");
try
{
string strippedResponse = serverResponse.Remove(0, startpoint);
ExtractEmails(strippedResponse);
}
catch { }
}
}
}
Assuming ScrapeBox.Lines implements IEnumerable, I would reccomend using Parallel.ForEach and passing the ScrapeBox.Lines as the IEnumerable over which to iterate.
Now, there is one additional problem, the code for reading the response from the HttpWebRequest still needs to write its output to a shared location. To accomplish that in a thread safe manner. A common way to do this is with a semaphore. You need a object accessible to each thread instance. A class level private variable private object sharedMutex = new object(); would work. Then the code ExtractEmails(strippedResponse); should be changed to
lock(sharedMutex)
{
ExtractEmails(strippedResponse);
}
Without having the code for the ExtractEmails(<string>) method, I can't provide a thread safe implementation for that, so that part of the solution may still cause a problem.