c# Reading source code with HttpWebRequest - c#

My code is as below. But I am getting error. How can I read the url source code in the example.
Error Message 'The remote server returned an error: (403) Forbidden'
CookieContainer cookierJar = new CookieContainer();
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create("https://www.hepsiburada.com/dyson-v11-absolute-extra-kablosuz-supurge-dyson-turkiye-garantili-p-HBV00000W4W6U");
webRequest.AllowAutoRedirect = true;
webRequest.CachePolicy = new System.Net.Cache.RequestCachePolicy(System.Net.Cache.RequestCacheLevel.NoCacheNoStore);
webRequest.CookieContainer = cookierJar;
webRequest.UserAgent = "My Thirsty Browser";
int statusCode = 0;
string sourceCode = string.Empty;
try
{
HttpWebResponse webResponse = (HttpWebResponse)webRequest.GetResponse();
statusCode = (int)webResponse.StatusCode;
StreamReader readContent = new StreamReader(webResponse.GetResponseStream());
sourceCode = readContent.ReadToEnd();
webResponse.Close();
webResponse = null;
}
catch (WebException xc)
{
if (xc.Response is HttpWebResponse)
{
HttpWebResponse rs = xc.Response as HttpWebResponse;
StreamReader readContent = new StreamReader(rs.GetResponseStream());
if (readContent != null)
{
sourceCode = readContent.ReadToEnd();
}
statusCode = (int)rs.StatusCode;
}
else
{
statusCode = (int)xc.Status;
sourceCode = xc.Message;
}
}
catch (Exception xc)
{
sourceCode = xc.Message;
}

I get a 403 in IE, I guess you need to be logged in to retrieve the resource. Your browser may have the credentials cached but your app isn't designed to log you in. Or are you logged in to Google in your browser - try logging out and see if you still have access....

Related

ALM Rest API : site-session returns 'The remote server returned an error: (400) Bad Request.'

I am getting '(400) Bad Request.' when I try complete authenticate against an ALM REST API, the first part (authentication) is successful) and I get the LWSSO_COOKIE_KEY, but site-session always fails with a 400 error code.
What am I doing wrong please... very confused!
// Authentication XML : 0 = User, 1 = Password
private const string AuthenticationXML = #"<alm-authentication>" +
"<user>{0}</user><password>{1}</password></alm-authentication>";
baseRequestURL = settings.QualityCentreURL + "/qcbin/";
Authentication is done first (and is successful) :
string authRequest = baseRequestURL + "authentication-point/alm-authenticate";
HttpWebRequest myauthrequest = (HttpWebRequest)WebRequest.Create(authRequest);
string xml = String.Format(AuthenticationXML, qcSettings.Username, qcSettings.Password);
byte[] Requestbytes = Encoding.UTF8.GetBytes(xml);
myauthrequest.Method = "POST";
myauthrequest.ContentType = "application/xml";
myauthrequest.ContentLength = Requestbytes.Length;
myauthrequest.Accept = "application/xml";
Stream RequestStr = myauthrequest.GetRequestStream();
RequestStr.Write(Requestbytes, 0, Requestbytes.Length);
RequestStr.Close();
HttpWebResponse myauthres = (HttpWebResponse)myauthrequest.GetResponse();
authenticationCookie = myauthres.Headers.Get("Set-Cookie");
The Site-Session code is :
public void GetSiteSession()
{
// Creat the web request fore site-session.
string request = baseRequestURL + "rest/site-session";
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(request);
string xml = String.Empty;
byte[] requestbytes = Encoding.UTF8.GetBytes(xml);
// Update the attributes before sending.
webRequest.Method = "POST";
webRequest.ContentType = "application/xml";
webRequest.Accept = "application/xml";
webRequest.Headers.Set(HttpRequestHeader.Cookie, authenticationCookie);
try
{
Stream requestStream = webRequest.GetRequestStream();
requestStream.Write(requestbytes, 0, requestbytes.Length);
requestStream.Close();
HttpWebResponse webRequestResponse = (HttpWebResponse)webRequest.GetResponse();
Stream responseStream = webRequestResponse.GetResponseStream();
XDocument doc = XDocument.Load(responseStream);
}
catch (System.Net.WebException except)
{
Console.WriteLine(except.Message);
}
}
I have tried cutting ;Path=/;HTTPOnly from LWSSO_COOKIE_KEY as per this question, but to no avail.
The API reference I found(here) seems to be a big vague or, possibly that I haven't understood it... :P
Apologies, it seems that with 12.53 I should have been using 'api/authentication/sign-in'
string requestURL = baseRequestURL + "api/authentication/sign-in";
try
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(requestURL);
var credentials = String.Format("{0}:{1}", qcSettings.Username, qcSettings.Password);
request.CookieContainer = authenticationCookieContainer;
request.Headers.Set(HttpRequestHeader.Authorization, "Basic " + Convert.ToBase64String(Encoding.UTF8.GetBytes(credentials)));
var authResponse = request.GetResponse();
errorString = String.Empty;
}
catch (System.Net.WebException except)
{
errorString = except.Message;
return false;
}
errorString = String.Empty;
return true;
}

C# - WebRequest.GetResponce(): The request was aborted: The connection was closed unexpectedly

i'm having a problem with this simple code which sends a request using provided url and reads html from responce. Looks like it's something with encoding of cyrillic symbols after ?q=, but i can't see why (url was actually obtained from browser address bar, not generated or anything else).
url =
"http://www.avito.ru/nizhniy_novgorod/kvartiry/sdam/na_dlitelnyy_srok?q=%D0%93%D0%B5%D0%BD%D0%B5%D1%80%D0%B0%D0%BB%D0%B0+%D0%98%D0%B2%D0%BB%D0%B8%D0%B5%D0%B2%D0%B0+10%D0%BA1";
string html = "";
try
{
Uri uri = new Uri(url);
WebRequest request = WebRequest.Create(uri);
request.Timeout = 100000;
using (WebResponse responce = request.GetResponse())
{
Stream stream = responce.GetResponseStream();
StreamReader reader = new StreamReader(stream);
html = reader.ReadToEnd();
}
}
catch (Exception e)
{
System.Console.WriteLine(e.Message);
}
Error occures in GetResponce() method. The message is:
The request was aborted: The connection was closed unexpectedly.
You should cast your request and response to HttpWebRequest and HttpWebResponse, respectively.
var url = "http://www.avito.ru/nizhniy_novgorod/kvartiry/sdam/na_dlitelnyy_srok?q=%D0%93%D0%B5%D0%BD%D0%B5%D1%80%D0%B0%D0%BB%D0%B0+%D0%98%D0%B2%D0%BB%D0%B8%D0%B5%D0%B2%D0%B0+10%D0%BA1";
string html = "";
try
{
Uri uri = new Uri(uri);
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
request.Timeout = 100000;
using (HttpWebResponse responce = (HttpWebResponse)request.GetResponse())
{
Stream stream = responce.GetResponseStream();
StreamReader reader = new StreamReader(stream);
html = reader.ReadToEnd();
Console.WriteLine(html);
}
}
catch (Exception e)
{
System.Console.WriteLine(e.Message);
}
Also, It seems that the url http://www.avito.ru/nizhniy_novgorod/kvartiry/sdam/na_dlitelnyy_srok?q=%D0%93%D0%B5%D0%BD%D0%B5%D1%80%D0%B0%D0%BB%D0%B0+%D0%98%D0%B2%D0%BB%D0%B8%D0%B5%D0%B2%D0%B0+10%D0%BA1 is invalid.
Using Fiddler, the url returns a 404 error.

Login to other web site from ASP .NET MVC 5 programmatically with Username & Password

I am working on ASP .Net MVC 5 and want to login programmatically to other web site (server side) and access its source page after log in. I did POST request as shown in below code with "OK" response but the page source always showing that I need to login.
//Code on controller
private static string LoginOn(string url, string email, string password)
{
try
{
var http = WebRequest.Create(url) as HttpWebRequest;
http.KeepAlive = true;
http.Method = "POST";
http.ContentType = "application/x-www-form-urlencoded";
var dataBytes = Encoding.UTF8.GetBytes($"vb_login_username={email}&vb_login_password={password}");
http.ContentLength = dataBytes.Length;
//http.AllowAutoRedirect = false;
using (var postStream = http.GetRequestStream())
{
postStream.Write(dataBytes, 0, dataBytes.Length);
}
var httpResponse = http.GetResponse() as HttpWebResponse;
var source = string.Empty;
using (StreamReader sr = new StreamReader(httpResponse.GetResponseStream()))
{
source = sr.ReadToEnd(); //I debug source here
}
return httpResponse?.ResponseUri.ToString();
//http = WebRequest.Create(httpResponse?.ResponseUri.ToString()) as HttpWebRequest;
//http.CookieContainer = new CookieContainer();
//http.CookieContainer.Add(httpResponse.Cookies);
//HttpWebResponse httpResponse2 = http.GetResponse() as HttpWebResponse;
//using (StreamReader sr = new StreamReader(httpResponse2.GetResponseStream()))
//{
// source = sr.ReadToEnd();
//}
//return httpResponse2?.ResponseUri.ToString();
}
catch (Exception e)
{
Log(e);
}
return null;
}

httpwebrequest gives timeout until restarted

I am working on a desktop application developed in C# (.NET environment).
This application connects to remote server using HttpWebRequest. If due to any reason my PC is disconnected from the internet and I re-connect it my application always gives request timeout for HttpWebRequest until I restart my whole application and if I again add new thread to my application after network d/c it works fine.
Is there any way to reset my network or anyone can tell me how does it work?
//my code is..
public String request(String add, String post, int time, String reff, int id, int rwtime)
{
try
{
if (rwtime == 0)
{
rwtime = 100000;
}
string result = "";
string location = "";
// Create the web request
HttpWebRequest req = WebRequest.Create(add) as HttpWebRequest;
req.ReadWriteTimeout = rwtime;
req.KeepAlive = true;
req.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate");
req.Accept = "application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5";
req.ContentType = "application/x-www-form-urlencoded";
req.Timeout = time;
req.Referer = reff;
req.AllowAutoRedirect = false;
req.CookieContainer = statictk.cc[id];
req.PreAuthenticate = true;
if (post != "")
{
req.Method = "POST";
string postData = post;
ASCIIEncoding encoding = new ASCIIEncoding();
byte[] byte1 = encoding.GetBytes(postData);
// Set the content type of the data being posted.
req.ContentType = "application/x-www-form-urlencoded";
// Set the content length of the string being posted.
req.ContentLength = byte1.Length;
Stream newStream = req.GetRequestStream();
newStream.Write(byte1, 0, byte1.Length);
newStream.Close();
}
else
{
req.Method = "GET";
}
// Get response
try
{
HttpWebResponse response = req.GetResponse() as HttpWebResponse;
// Get the response stream
location = response.GetResponseHeader("Location");
if (location == "")
{
Stream responseStream = response.GetResponseStream();
if (response.ContentEncoding.ToLower().Contains("gzip"))
responseStream = new GZipStream(responseStream, CompressionMode.Decompress);
else if (response.ContentEncoding.ToLower().Contains("deflate"))
responseStream = new DeflateStream(responseStream, CompressionMode.Decompress);
StreamReader reader = new StreamReader(responseStream, Encoding.Default);
// Read the whole contents and return as a string
result = reader.ReadToEnd();
}
else
{
result = location;
}
response.Close();
if (result == "") result = "retry";
return result;
}
catch (Exception e)
{
log.store("errorinresponce", e.Message);
if (statictd.status[id] != "removed")
{
return "retry";
}
else
{
return "error";
}
}
}
catch(Exception f)
{
log.store("Networkerrorretry", f.Message);
if (f.Message == "The operation has timed out")
{
return "retry";
}
string ans = MessageBox.Show("There was a Network Error..Wish to Retry ?\nError msg : "+ f.Message, "Title", MessageBoxButtons.YesNo).ToString();
if (ans == "Yes")
return "retry";
else
{
Invoketk.settxt(id, "Not Ready");
return "error";
}
}
}
It sounds like your application is missing some error handling. A disconnect can happen at any time and your application should be able to handle it. Try to surround the network loop with a try-catch statement, and then catch for the different kinds of exceptions. Depending on what exception was thrown, you can then decide if you reconnect to the server silently or if you want to generate an error message.

How to check if remote image exists with C#

public void BuildImg()
{
// The two different images as strings.
string url1 = "http://remoteimage.com/image.jpg";
string url2 = "http://remoteimage.com/image2.jpg";
try
{
// Check to see if url1 exists or not
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url1);
request.Credentials = System.Net.CredentialCache.DefaultCredentials;
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
myImg.Visible = true;
myImg.ImageUrl = url1;
}
catch (Exception ex)
{
// Check to see if url2exists or not
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url2);
request.Credentials = System.Net.CredentialCache.DefaultCredentials;
HttpWebResponse response;
try
{
response = request.GetResponse() as HttpWebResponse;
}
catch (WebException exc)
{
response = exc.Response as HttpWebResponse;
}
// Set myImg to show if url2 exists
myImg.Visible = true;
myImg.ImageUrl = url2;
// If response returns 404, then hide myImg
if (response.StatusCode == HttpStatusCode.NotFound)
{
myImg.Visible = false;
}
}
var arr = new[]
{
"http://example.com/image.jpg",
"http://example.com/image2.jpg"
...
};
myImg.ImageUrl = arr.FirstOrDefault(i => CheckExistence(i));
static bool CheckUrlExistence(string url)
{
try
{
var request = (HttpWebRequest)WebRequest.Create(url);
request.Credentials = CredentialCache.DefaultCredentials;
request.Method = "HEAD";
var response = (HttpWebResponse)request.GetResponse();
return response.StatusCode == HttpStatusCode.OK;
}
catch (Exception ex)
{
var code = ((HttpWebResponse)((WebException)ex).Response).StatusCode; // NotFound, etc
return false;
}

Categories