This question already has answers here:
Login to the page with HttpWebRequest
(2 answers)
Closed 6 years ago.
I need some help with a work project I have been assigned. At the moment we manually go to the site, logon and then download 2 excel files from a supplier's website every month. The files are then loaded into SQL.
We want to automate this process. Now the loading of the files into SQL I can do, but I am not sure how I can automate logging onto the website entering my user details and collecting the files. I mostly deal with SQL and have very little .NET experience, so any code samples would be most appreciated.
Just to confirm. The logon form is on a aspx page. just a basic form with a table containing the username & password fields, the forgotten password link and the logon button
You can either use webclient or httpwebrequest.
Login to the page with HttpWebRequest
How do you login to a webpage and retrieve its content in C#?
Httpwebrequest example:
HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create("http://sso.bhmobile.ba/sso/login");
req.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.0.3705;)";
req.Method = "POST";
req.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
req.Headers.Add("Accept-Language: en-us,en;q=0.5");
req.Headers.Add("Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7");
req.KeepAlive = true;
req.Headers.Add("Keep-Alive: 300");
req.Referer ="http://sso.bhmobile.ba/sso/login";
req.ContentType = "application/x-www-form-urlencoded";
String Username = "username";
String PassWord = "Password";
StreamWriter sw = new StreamWriter(req.GetRequestStream());
sw.Write("application=portal&url=http%3A%2F%2Fwww.bhmobile.ba%2Fportal%2Fredirect%3Bjsessionid%3D1C568AAA1FB8B5C757CF5F68BE6ECE65%3Ftype%3Dssologin%26url%3D%2Fportal%2Fshow%3Bjsessionid%3D1C568AAA1FB8B5C757CF5F68BE6ECE65%3Fidc%3D1023278&realm=sso&userid=" + Username + "&password=" + password + "&x=16&y=11");
sw.Close();
HttpWebResponse response = (HttpWebResponse)req.GetResponse();
StreamReader reader = new StreamReader(response.GetResponseStream());
string tmp = reader.ReadToEnd();
foreach (Cookie cook in response.Cookies)
{
tmp += "\n" + cook.Name + ": " + cook.Value;
}
Response.Write(tmp);
Response.End();
Webclient example:
WebClient wc = new WebClient();
wc.Credentials = new NetworkCredential("username", "password");
string url = "http://foo.com";
try
{
using (Stream stream = wc.OpenRead(new Uri(url)))
{
using (StreamReader reader = new StreamReader(stream))
{
return reader.ReadToEnd();
}
}
}
catch (WebException e)
{
//Error handeling
}
Related
I have a case with a simple application that browses to a XenForo Forum site, fetches cookies and further sends post data to login and fetch some information only available for logged in users.
I am able to successfully fetch the cookies and verify that i am successfully logged in the first time, but i cant seem to stay logged in when i try to "browse" further when trying to reuse the same cookies.
Here is what i got so far:
public MainWindow()
{
InitializeComponent();
if (IsLoggedIn())
{
GetPage("http://thesiteiloginto.org/someotherpage");
}
}
// Store Cookies
CookieCollection Cookies;
void GetCookies(string cookieUrl)
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(cookieUrl);
request.CookieContainer = new CookieContainer();
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
using (Stream responseStream = response.GetResponseStream())
{
// Store Cookies
Cookies = response.Cookies;
}
}
bool IsLoggedIn()
{
GetCookies(_cookieUrl);
CookieCollection cookies = Cookies;
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(loginUrl);
request.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)";
request.CookieContainer = new CookieContainer();
request.Method = "POST";
request.ContentType = "application/x-www-form-urlencoded";
string postData = "login=" + username +
"&password=" + password +
"&_xfToken=" + xfToken +
"&cookie_check=" + cookie_check +
"&redirect=" + redirect;
byte[] bytes = Encoding.UTF8.GetBytes(postData);
request.ContentLength = bytes.Length;
if (cookies != null)
{
Console.WriteLine("Cookies are present");
Console.WriteLine(Cookies.Count);
Console.WriteLine(Cookies[0].Value);
request.CookieContainer.Add(cookies);
}
using (Stream requestStream = request.GetRequestStream())
{
requestStream.Write(bytes, 0, bytes.Length);
WebResponse response = request.GetResponse();
using (Stream stream = response.GetResponseStream())
{
using (StreamReader reader = new StreamReader(stream))
{
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(reader.ReadToEnd());
bool _loggedIn = false;
try
{
var uniqueNodeTest = doc.DocumentNode.SelectSingleNode("//*[#id=\"navigation\"]/div/nav/div/ul[2]/li[1]/a/strong[1]");
if (uniqueNodeTest.InnerText.Trim().ToLower() == uniqueNodeName)
{
Console.WriteLine("Logged in");
_loggedIn = true;
}
}
catch (Exception ex)
{
Console.WriteLine("Ops! [Login] # SelectSingleNode\n" + ex.Message);
_loggedIn = false;
}
return _loggedIn;
}
}
}
}
void GetPage(string url)
{
if (Cookies != null)
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
request.PreAuthenticate = true;
request.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)";
request.CookieContainer = new CookieContainer();
request.CookieContainer.Add(Cookies);
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
using (Stream responseStream = response.GetResponseStream())
{
using (StreamReader reader = new StreamReader(responseStream))
{
var pageSource = reader.ReadToEnd();
// Retuns false, meaning i am not logged in
//Where do i go from here?
Console.WriteLine(pageSource.Contains("Log Out"));
}
}
}
}
Console:
Cookies are present
1
22a6c5a4c5557a7f7db36f50a1d746f1
Logged in
False
As you can see i am logged in after the first test, but i cant seem to stay logged in when trying to further browse reusing the cookies.
What am i not taking into consideration? How can i stay logged in to the site?
Use CookieContainer as class to store them locally.
When retrieving the response, put all your cookies into the CookieContainer. When preparing the request, just set the request.CookieContainer to your own object on the new call.
This is the code I used for saving my cookies in a project of mine. Since it's part of one fully decked out HttpSession class specifically meant for the kind of requests you're doing, you'll notice both the response and the cookie container are class variables.
/// <summary>
/// Fetches the new cookies and saves them in the cookie jar.
/// </summary>
private void SaveNewCookies()
{
try
{
foreach (Cookie c in this.m_HttpWebResponse.Cookies)
{
if (c.Domain.Length > 0 && c.Domain[0] == '.')
c.Domain = c.Domain.Remove(0, 1);
this.m_CookieJar.Add(new Uri(this.m_HttpWebResponse.ResponseUri.Scheme + "://" + c.Domain), c);
}
if (this.m_HttpWebResponse.Cookies.Count > 0)
this.BugFixCookieDomain(this.m_CookieJar);
}
catch
{
// no new cookies
}
}
As you see, this contains some smaller bug fixes as well. The mentioned BugFixCookieDomain function is a fix specifically for the 3.5 framework, which you can find here, but if you've moved past that to 4.0 and beyond it won't be particularly useful to you.
I created an windows form application that goes to a website and does a HTTPwebrequest POST and then displays the result in a webbrowser. I tested it on the computer that I wrote the program on and it works exactly like it should. However when I get the .exe file from the bin folder along with htmlagilitypack.dll and run a test on another computer, it does the POST data but web page that shows up in broswer is just the default login page. I have looked at it on fiddler and it seems like the cookies aren't getting set to the webbrowser. How can I fix this? I have also compiled the program in release mode and transferred the files in the release folder and its the same result.
On the program I used this to set the cookie for the browser.
[DllImport("wininet.dll", CharSet = CharSet.Auto, SetLastError = true)]
static extern bool InternetSetCookie(string lpszUrlName, string lpszCookieName, string lpszCookieData);
Does it have to do something with that dll file?
Edit: I have made sure both computer has 4.5 NET installed.
//get the cookie first
CookieCollection cookies = new CookieCollection();
CookieContainer cookiecontainer = new CookieContainer();
HttpWebRequest request = (HttpWebRequest)WebRequest.Create("http://www.supremenewyork.com" + url);
request.CookieContainer = new CookieContainer();
request.CookieContainer.Add(cookies);
//get the auth token
using (StreamReader authreader = new StreamReader(request.GetResponse().GetResponseStream()))
{
source = authreader.ReadToEnd();
}
//need the auth token
string token = Regex.Match(source, "authenticity_token.+?value=\"(.+?)\"").Groups[1].Value;
//need the POST url
string action = Regex.Match(source, "UTF-8.+?action=\"(.+?)\"").Groups[1].Value;
//get the reponse from the server and save the cookies from the first request
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
cookies = response.Cookies;
response.Close();
string formparam = string.Format("utf8=%E2%9C%93&authenticity_token={0}&size={1}&commit=add to cart", token, sizechart[0]);
HttpWebRequest webreq = (HttpWebRequest)WebRequest.Create("http://www.supremenewyork.com"+ action);
webreq.CookieContainer = new CookieContainer();
webreq.CookieContainer = request.CookieContainer;
//webreq.CookieContainer.Add(cookies); //recover the cookie first request
webreq.Method = "POST"; //set a POST method
webreq.Referer = "http://www.supremenewyork.com" + url;
webreq.ContentType = "application/x-www-form-urlencoded";
webreq.UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36";
webreq.KeepAlive = true;
webreq.AllowAutoRedirect = true;
byte[] bytes = Encoding.UTF8.GetBytes(formparam);
webreq.ContentLength = bytes.Length;
//write
Stream postdata = webreq.GetRequestStream(); //open connection
postdata.Write(bytes, 0, bytes.Length); //send the data
postdata.Close();
//get the final response from the server
HttpWebResponse resp = (HttpWebResponse)webreq.GetResponse();
cookies = resp.Cookies;
response.Close();
//Stream answer = resp.GetResponseStream();
//StreamReader _answer = new StreamReader(webreq.GetResponse().GetResponseStream());
//string reply = _answer.ReadToEnd();
//need to check if item has been added to cart
//richTextBox1.Text = reply;
//check if item has been added to cart
HttpWebRequest webreq2 = (HttpWebRequest)WebRequest.Create("http://www.supremenewyork.com/shop/cart");
//webreq2.CookieContainer = new CookieContainer();
webreq2.CookieContainer = webreq.CookieContainer;
HttpWebResponse resp2 = (HttpWebResponse)webreq2.GetResponse();
Stream answer2 = resp2.GetResponseStream();
StreamReader _answer2 = new StreamReader(webreq2.GetResponse().GetResponseStream());
string reply2 = _answer2.ReadToEnd();
string item = textBoxkeyword.Text;
string color = textBoxcolor.Text.ToLower();
if(reply2.Contains(color))
{
//proceed to check out
//update user
appendtext(nDateTime + item + " " + color + " added to cart" );
appendtext(nDateTime + "Please check out the item");
HttpWebRequest webreq3 = (HttpWebRequest)WebRequest.Create("http://www.supremenewyork.com/checkout");
//webreq3.CookieContainer = new CookieContainer();
webreq3.CookieContainer = webreq2.CookieContainer;
HttpWebResponse resp3 = (HttpWebResponse)webreq3.GetResponse();
//webBrowser1.ScriptErrorsSuppressed = false;
//RegistryKey RegKey = Registry.CurrentUser.OpenSubKey(#"Software\Microsoft\Internet Explorer\Main", true);
//RegKey.SetValue("Display Inline Images", "yes");
string cookie_string = "";
foreach (Cookie cook in resp3.Cookies)
{
cookie_string += cook.ToString() + ";";
InternetSetCookie("http://www.supremenewyork.com/checkout", cook.Name, cook.Value);
}
webBrowser1.Navigate("http://www.supremenewyork.com/checkout");
}
Edit2: added all the code
I want to display image recieved from webresponse to browser directly without saving to a file.i have used below code to save the image to file but i want to display in browser directly.the website provides the captcha image in webresponse which i want to display.please help me.
public void captcha(string id, string pass)
{
HttpWebRequest req;
HttpWebResponse response;
string strNewValue,ckuser,ckpass;
System.Drawing.Image returnval;
ckuser = id;
ckpass = pass;
this.req = (HttpWebRequest)WebRequest.Create("http://site2sms.com/security/captcha.asp");
ServicePointManager.Expect100Continue = false;
this.req.CookieContainer = new CookieContainer();
this.req.AllowAutoRedirect = false;
this.req.UserAgent = "Mozilla/5.0 (Windows NT 6.1; rv:8.0) Gecko/20100101 Firefox/8.0";
this.req.Accept = "*/*";
this.req.Method = "POST";
this.req.CookieContainer = cookieCntr;
this.req.ContentType = "application/x-www-form-urlencoded";
this.req.Referer = "http://site2sms.com/verification.asp?source=login";
this.strNewValue = "user=" + ckuser + "&password=" + ckpass + "&Submit=Sign+in";
this.req.ContentLength = this.strNewValue.Length;
StreamWriter writer = new StreamWriter(this.req.GetRequestStream(), Encoding.ASCII);
writer.Write(this.strNewValue);
writer.Close();
this.response = (HttpWebResponse)this.req.GetResponse();
returnval = Image.FromStream(response.GetResponseStream());
// returnval.Save( Server.MapPath("captcha.bmp"));
Response.AppendHeader("Content-Disposition:", "inline;filename=captcha.bmp");
Response.ContentType = "image/bmp";
Response.Write(returnval);
Response.End();
this.response.Close();
}
You can use HttpResponse.WriteFile method to send the stream to client directly.
Writes the specified file directly to an HTTP response output stream.
You can create the IHttpHandler to send the stream, thus avoid some page life circle, increase the performance. Here is the link
I need to login to a website a download the source code from various pages when logged in. I am able to do this quite easily when using the Windows Forms WebBrowser class, however this is not appropiate and I need to be able to do this with WebRequest or one of the others. Unfortunately it doesn't like how I am handling the cookies.
I am using the following code and get the following response: {"w_id":"LoginForm2_6","message":"Please enable cookies in your browser so you can log in.","success":0,"showLink":false}
string url2 = "%2Fapp%2Futils%2Flogin_form%2Fredirect%2Fhome";
string login = "username";
string password = "password";
string w_id = "LoginForm2_6";
string loginurl = "http://loginurl.com";
string cookieHeader;
WebRequest req = WebRequest.Create(loginurl);
req.Proxy = WebRequest.DefaultWebProxy;
req.Credentials = System.Net.CredentialCache.DefaultNetworkCredentials;
req.Proxy.Credentials = System.Net.CredentialCache.DefaultNetworkCredentials;
req.ContentType = "application/x-www-form-urlencoded; charset=UTF-8";
req.Method = "POST";
string postData = string.Format("w_id={2}&login={0}&password={1}&url2={3}", login, password, w_id, url2);
byte[] bytes = Encoding.ASCII.GetBytes(postData);
req.ContentLength = bytes.Length;
using (Stream os = req.GetRequestStream())
{
os.Write(bytes, 0, bytes.Length);
}
WebResponse resp = req.GetResponse();
cookieHeader = resp.Headers["Set-cookie"];
string pageSource = "";
using (StreamReader sr = new StreamReader(resp.GetResponseStream()))
{
pageSource = sr.ReadToEnd();
}
richTextBox1.Text = pageSource;
If anyone could tell me where I'm going wrong, it would be greatly appreciated.
Also, to let you know, if I use the following with the webbrowser class, it works in fine:
b.Navigate(fullurl, "", enc.GetBytes(postData), "Content-Type: application/x-www-form-urlencoded\r\n");
I know this is old to reply, but the user Matthew Brindley answered similar question with a completely working example. The question is about accessing to the
source code of a website that requires user login previously. All done from a C# application using WebRequest and WebResponse
Okay I tried asking this question yesterday but i'm not sure if I gave enough info, i got an answer but it hasn't worked for me. Basically what i'm doing is the user opens this windows forms application and logs in. Afterwhich they enter some text into a textbox and click run. At this point the run function is making a webrequest to a server that requires a login (the login that is initially done after they open the program. For some reason its still not seeing that the user is logged in when performing the second request even though the cookies are added too a cookie container. I'm not sure what i'm doing wrong but I will post my code so you can further help me.
This is the function that is performed for the login when the user enters the application.
private void button1_Click(object sender, EventArgs e)
{
string paramaters = "authmethod=on&chkRememberMe=on&login-form-type=pwd&password=" + pw.Text + "&userid=" + uid.Text + "&username=" + uid.Text;
string strResponse;
HttpWebRequest requestLogin = (HttpWebRequest)WebRequest.Create("https://www.url.com/login.form");
requestLogin.Method = "POST";
requestLogin.CookieContainer = cookieJar;
requestLogin.ContentType = "application/x-www-form-urlencoded";
requestLogin.ContentLength = paramaters.Length;
StreamWriter stOut = new StreamWriter(requestLogin.GetRequestStream(), System.Text.Encoding.ASCII);
stOut.Write(paramaters);
stOut.Close();
HttpWebResponse responseLogin = (HttpWebResponse)requestLogin.GetResponse();
StreamReader stIn = new StreamReader(responseLogin.GetResponseStream());
strResponse = stIn.ReadToEnd();
stIn.Close();
//Add cookies to CookieJar (Cookie Container)
foreach (Cookie cookie in responseLogin.Cookies)
{
cookieJar.Add(new Cookie(cookie.Name.Trim(), cookie.Value.Trim(), cookie.Path, cookie.Domain));
richTextBox2.Text += cookie.Name.ToString() + Environment.NewLine + cookie.Value.ToString() + Environment.NewLine + cookie.Path.ToString() + Environment.NewLine + cookie.Domain.ToString();
}
if (strResponse.Contains("Log On Successful") || strResponse.Contains("already has a webseal session"))
{
foreach (Control cont in this.Controls)
{
cont.Visible = true;
}
loginPanel.SendToBack();
loginPanel.Visible = false;
}
else
{
MessageBox.Show("Login failed.");
}
}
This is the function that is ran when the user clicks the "run" button to initiate the tests on a consumer account.
private string runTestRequest(Uri url, string parameters)
{
string testResults = string.Empty;
HttpWebRequest runTest = (HttpWebRequest)WebRequest.Create(url);
runTest.CookieContainer = cookieJar;
runTest.Method = "POST";
runTest.ContentType = "application/x-www-form-urlencoded";
StreamWriter stOut = new StreamWriter(runTest.GetRequestStream(), System.Text.Encoding.ASCII);
stOut.Write(parameters);
stOut.Close();
StreamReader stIn = new StreamReader(runTest.GetResponse().GetResponseStream());
testResults = stIn.ReadToEnd();
stIn.Close();
return testResults;
}
And of course this is my cookie container object
public CookieContainer cookieJar = new CookieContainer();
P.S.: The domains of the webrequests are different. First being abc.com 2nd being 123.com The only problem is that the first domain (which is the login) is a global login for internal web applications like 123.com, so how would i use the login session from the 1st domain with the 2nd domain?
Can you please assist in helping me figure out what I am doing wrong.
I found out that what was happening was it was redircting to a subdomain on the same domain as the 2nd (123.com) to use the login. Evidently they had this global login system built on the multiple domains to pass the cookies. The code above DOES work and i do have it working now. Thanks!!
string url = "http://www.ABC/MemberShip/Login.aspx";// HttpContext.Current.Request.Url.AbsoluteUri.ToString().Replace("AutoLogin", "Login");
CookieContainer myCookieContainer = new CookieContainer();
HttpWebRequest request = WebRequest.Create(url) as HttpWebRequest;
request.CookieContainer = myCookieContainer;
request.Method = "GET";
request.KeepAlive = false;
HttpWebResponse response = request.GetResponse() as HttpWebResponse;
System.IO.Stream responseStream = response.GetResponseStream();
System.IO.StreamReader reader = new System.IO.StreamReader(responseStream, Encoding.UTF8);
string srcString = reader.ReadToEnd();
// get the page ViewState
string viewStateFlag = "id=\"__VIEWSTATE\" value=\"";
int i = srcString.IndexOf(viewStateFlag) + viewStateFlag.Length;
int j = srcString.IndexOf("\"", i);
string viewState = srcString.Substring(i, j - i);
// get page EventValidation
string eventValidationFlag = "id=\"__EVENTVALIDATION\" value=\"";
i = srcString.IndexOf(eventValidationFlag) + eventValidationFlag.Length;
j = srcString.IndexOf("\"", i);
string eventValidation = srcString.Substring(i, j - i);
string submitButton = "LoginButton";
// UserName and Password
string userName = "userid";
string password = "password";
// Convert the text into the url encoding string
viewState = System.Web.HttpUtility.UrlEncode(viewState);
eventValidation = System.Web.HttpUtility.UrlEncode(eventValidation);
submitButton = System.Web.HttpUtility.UrlEncode(submitButton);
// Concat the string data which will be submit
string formatString =
"txtUserName={0}&txtPassword={1}&btnSignIn={2}&__VIEWSTATE={3}&__EVENTVALIDATION={4}";
string postString =
string.Format(formatString, userName, password, submitButton, viewState, eventValidation);
// Convert the submit string data into the byte array
byte[] postData = Encoding.ASCII.GetBytes(postString);
// Set the request parameters
request = WebRequest.Create(url) as HttpWebRequest;
request.Method = "POST";
request.Referer = url;
request.KeepAlive = false;
request.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; CIBA)";
request.ContentType = "application/x-www-form-urlencoded";
request.CookieContainer = myCookieContainer;
System.Net.Cookie ck = new System.Net.Cookie("TestCookie1", "Value of test cookie");
ck.Domain = request.RequestUri.Host;
request.CookieContainer.Add(ck);
request.CookieContainer.Add(response.Cookies);
request.ContentLength = postData.Length;
// Submit the request data
System.IO.Stream outputStream = request.GetRequestStream();
request.AllowAutoRedirect = true;
outputStream.Write(postData, 0, postData.Length);
outputStream.Close();
// Get the return data
response = request.GetResponse() as HttpWebResponse;
responseStream = response.GetResponseStream();
reader = new System.IO.StreamReader(responseStream, Encoding.UTF8);
srcString = reader.ReadToEnd();
Response.Write(srcString);
Response.End();