I would like to be able to simulate a browser engine from my code to retrieve a web page from a specific link, and parse the data, the problem is that the page contains some Ajax calls and all I can get is the source code before any script is running.
Is there an easy way to get the final page from the code after the?
I'm currently using the following code:
//create the constructor with post type and few data
MyWebRequest myRequest = new MyWebRequest("https://www.kayak.com/flights/LHR-AMS/2014-09-10/2014-10-15");
//show the response string on the console screen.
Response.Write(myRequest.GetResponse());
public class MyWebRequest
{
private WebRequest request;
private Stream dataStream;
private string status;
public String Status
{
get
{
return status;
}
set
{
status = value;
}
}
public MyWebRequest(string url)
{
// Create a request using a URL that can receive a post.
request = WebRequest.Create(url);
}
public MyWebRequest(string url, string method)
: this(url)
{
if (method.Equals("GET") || method.Equals("POST"))
{
// Set the Method property of the request to POST.
request.Method = method;
}
else
{
throw new Exception("Invalid Method Type");
}
}
public MyWebRequest(string url, string method, string data)
: this(url, method)
{
// Create POST data and convert it to a byte array.
string postData = data;
byte[] byteArray = Encoding.UTF8.GetBytes(postData);
// Set the ContentType property of the WebRequest.
request.ContentType = "application/x-www-form-urlencoded";
// Set the ContentLength property of the WebRequest.
request.ContentLength = byteArray.Length;
// Get the request stream.
dataStream = request.GetRequestStream();
// Write the data to the request stream.
dataStream.Write(byteArray, 0, byteArray.Length);
// Close the Stream object.
dataStream.Close();
}
public string GetResponse()
{
// Get the original response.
WebResponse response = request.GetResponse();
this.Status = ((HttpWebResponse)response).StatusDescription;
// Get the stream containing all content returned by the requested server.
dataStream = response.GetResponseStream();
// Open the stream using a StreamReader for easy access.
StreamReader reader = new StreamReader(dataStream);
// Read the content fully up to the end.
string responseFromServer = reader.ReadToEnd();
// Clean up the streams.
reader.Close();
dataStream.Close();
response.Close();
return responseFromServer;
}
Related
I am currently working on implementing a class for accessing rest webservices with my C# program. I managed to do that for PUT, Post and GET, but I have problems with the Patch. The following error occurs everytime:
There is an exception error “System.Net.WebException” in System.ddll
Additional information: remote server reported: (400) Unvalid request
I have researched and tried out various things – to no avail. I would appreciate any help!
Many thanks in advance
public string WebserviceSenden()
{
// Create a request using a URL that can receive a post.
WebRequest request = WebRequest.Create( GrundURL + ErweiterungsURL);
// Set the Method property of the request to POST.
request.Method = Methode;
// Create POST data and convert it to a byte array.
string postData = DateSenden;
byte[] byteArray = Encoding.UTF8.GetBytes(postData);
// Set the ContentType property of the WebRequest.
request.ContentType = "application/json";
// Set the ContentLength property of the WebRequest.
request.ContentLength = byteArray.Length;
// Get the request stream.
Stream dataStream = request.GetRequestStream();
// Write the data to the request stream.
dataStream.Write(byteArray, 0, byteArray.Length);
// Close the Stream object.
dataStream.Close();
// Get the response.
WebResponse response = request.GetResponse();
// Display the status.
Console.WriteLine(((HttpWebResponse)response).StatusDescription);
// Get the stream containing content returned by the server.
dataStream = response.GetResponseStream();
// Open the stream using a StreamReader for easy access.
StreamReader reader = new StreamReader(dataStream);
// Read the content.
string responseFromServer = reader.ReadToEnd();
// Display the content.
Console.WriteLine(responseFromServer);
// Clean up the streams.
reader.Close();
dataStream.Close();
response.Close();
return responseFromServer;
}
public string WebserviceEmpfangen()
{
// Create a request for the URL.
WebRequest request = WebRequest.Create(
GrundURL + ErweiterungsURL);
// If required by the server, set the credentials.
request.Credentials = CredentialCache.DefaultCredentials;
// Get the response.
WebResponse response = request.GetResponse();
// Display the status.
Console.WriteLine(((HttpWebResponse)response).StatusDescription);
// Get the stream containing content returned by the server.
Stream dataStream = response.GetResponseStream();
// Open the stream using a StreamReader for easy access.
StreamReader reader = new StreamReader(dataStream);
// Read the content.
string responseFromServer = reader.ReadToEnd();
// Display the content.
Console.WriteLine(responseFromServer);
// Clean up the streams and the response.
reader.Close();
response.Close();
return responseFromServer;
}
}
private void button1_Click(object sender, EventArgs e)
{
String Jason = "";
RestClient AbfrageStarten = new RestClient();
AbfrageStarten.GrundURL = "URL";
AbfrageStarten.ErweiterungsURL = "540697";
Jason = AbfrageStarten.WebserviceEmpfangen();
var settings = new JsonSerializerSettings
{
NullValueHandling = NullValueHandling.Ignore,
MissingMemberHandling = MissingMemberHandling.Ignore
};
var list = JsonConvert.DeserializeObject<List<Lagereinheit>>(Jason, settings);
KorrekturLagereinheit = list[0];
KorrekturLagereinheit.stueckzahl = 5858;
string Test;
Test = JsonConvert.SerializeObject(KorrekturLagereinheit, Formatting.None, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore });
RestClient AntwortSenden = new RestClient();
AntwortSenden.GrundURL = "URL";
AntwortSenden.ErweiterungsURL = "540697";
AntwortSenden.Methode = "Patch";
AntwortSenden.DateSenden = Test;
AntwortSenden.WebserviceSenden();
}
Error message
Many thanks in advance
Im trying to send some data to a machin via LAN cable and WebRequest
this is the class im using
public class MyWebRequest
{
private WebRequest request;
private Stream dataStream;
private string status;
public String Status
{
get
{
return status;
}
set
{
status = value;
}
}
public MyWebRequest(string url)
{
request = WebRequest.Create(url);
}
public MyWebRequest(string url, string method)
: this(url)
{
if (method.Equals("GET") || method.Equals("POST"))
{
request.Method = method;
}
else
{
throw new Exception("Invalid Method Type");
}
}
public MyWebRequest(string url, string method, string data)
: this(url, method)
{
string postData = data;
byte[] byteArray = Encoding.UTF8.GetBytes(postData);
request.ContentType = "application/x-www-form-urlencoded";
// Set the ContentLength property of the WebRequest.
request.ContentLength = byteArray.Length;
dataStream = request.GetRequestStream();
dataStream.Write(byteArray, 0, byteArray.Length);
dataStream.Close();
}
public string GetResponse()
{
WebResponse response = request.GetResponse();
this.Status = ((HttpWebResponse)response).StatusDescription;
dataStream = response.GetResponseStream();
StreamReader reader = new StreamReader(dataStream);
string responseFromServer = reader.ReadToEnd();
reader.Close();
dataStream.Close();
response.Close();
return responseFromServer;
}
}
and the action code is
command="?insert_employee:cart_number=0007989222&name=علي&last_name=رمضاني&finger_number=123&?";
MyWebRequest wr = new MyWebRequest(command, "GET");
string Response = wr.GetResponse();
as you see there is some utf-8 character in my command and it dose not work both in browser and my application
but when i try this string it works perfectly:
command = "?insert_employee:cart_number=0007989222&name=ali&last_name=ramazani&finger_number=123&?"
i tested many ways which i found in stackoverflow such as "using uri instead of url", "using httpUtility.UrlEncode", ... but none of them was effective or maybe i use them in a wrong way.
do you have any idea for helping?
You can use the HttpUtility.UrlPathEncode method you encode the path portion of a URL in order to guarantee a consistent decoded URL, regardless of which platform or browser performs the decoding.
Whereas there is beautiful explanation of Encoding
urlPrueba = "http://someexample.com/?insert_employee:cart_number=0007989222&name=علي&last_name=رمضاني&finger_number=123&?";
var uri = new Uri(urlPrueba);
Console.WriteLine("urlPrueba" + " = " + uri.AbsoluteUri);
I'm trying to reopen resolved or closed Mantis issues using MantisConnect
I have thusfar succeeded in changing the status of non resolved/closed issues but when the issue is one of these two, i keep getting the following Exception:
System.ServiceModel.ProtocolException
with the message:
There is a problem with the XML that was received from the network.
See inner exception for more details.
with inner Exception:
System.Xml.XmlException
With the message:
The encoding in the declaration 'ISO-8859-1' does not match the encoding of the document 'utf-8'.
I'm using the update command to do the update described here: https://www.mantisbt.org/bugs/api/soap/mantisconnect.php?wsdl
got my mantisconnect code from github here:
https://github.com/mantishub/MantisDotNetClient
The funny thing is that the changes to other states and even to the resolved and closed state work, the Exception is only throws when the issue is already resolved or closed
I could not find any other commands in mantisconnect to reopen an issue, anyone have an idea.
Edit:
Mantis issue I was talking about in my comment The ticket: https://mantisbt.org/bugs/view.php?id=16579
Since there is a bug in the mantis API I solved this problem by navigating the website using WebRequests and the bulk UP_STATUS feature from mantis. It's dirty but it works.
class MantisWebRequest
{
public string Password { private get; set; }
public string Username { private get; set; }
public CookieContainer CookieContainer { get; private set; }
public StreamReader Reader { get; private set; }
public Stream DataStream { get; private set; }
public WebResponse Response { get; private set; }
public HttpWebRequest Request { get; private set; }
public string Page { get; private set; }
public MantisWebRequest(string username, string password)
{
Username = username;
Password = password;
CookieContainer = new CookieContainer();
}
public void Connect(string cookieName = null)
{
string loginurl = "http://mantispageurl/login.php";
// Create a request using a URL that can receive a post.
Request = (HttpWebRequest)System.Net.WebRequest.Create(loginurl);
// Set the Method property of the request to POST.
Request.Method = "POST";
Request.KeepAlive = true;
if (cookieName != null)
CookieContainer.Add(new Cookie(cookieName, Username, "/", new Uri(loginurl).Host));
Request.CookieContainer = CookieContainer;
// Create POST data and convert it to a byte array. Modify this line accordingly
string postData = String.Format("username={0}&password={1}&secure_session=on", Username, Password);
byte[] byteArray = Encoding.UTF8.GetBytes(postData);
// Set the ContentType property of the WebRequest.
Request.ContentType = "application/x-www-form-urlencoded";
// Set the ContentLength property of the WebRequest.
Request.ContentLength = byteArray.Length;
// Get the request stream.
DataStream = Request.GetRequestStream();
// Write the data to the request stream.
DataStream.Write(byteArray, 0, byteArray.Length);
// Close the Stream object.
DataStream.Close();
// Get the response.
Response = Request.GetResponse();
// Get the stream containing content returned by the server.
DataStream = Response.GetResponseStream();
// Open the stream using a StreamReader for easy access.
Reader = new StreamReader(DataStream);
// Read the content.
Page = Reader.ReadToEnd();
// Clean up the streams.
Reader.Close();
DataStream.Close();
Response.Close();
}
private void POST(string url, string postData)
{
Request = (HttpWebRequest)System.Net.WebRequest.Create(url);
Request.Method = "POST";
Request.KeepAlive = true;
Request.CookieContainer = CookieContainer;
byte[] byteArray = Encoding.UTF8.GetBytes(postData);
// Set the ContentType property of the WebRequest.
Request.ContentType = "application/x-www-form-urlencoded";
// Set the ContentLength property of the WebRequest.
Request.ContentLength = byteArray.Length;
// Get the request stream.
DataStream = Request.GetRequestStream();
// Write the data to the request stream.
DataStream.Write(byteArray, 0, byteArray.Length);
// Close the Stream object.
DataStream.Close();
Response = Request.GetResponse();
// Get the stream containing content returned by the server.
DataStream = Response.GetResponseStream();
// Open the stream using a StreamReader for easy access.
Reader = new StreamReader(DataStream);
// Read the content.
Page = Reader.ReadToEnd();
// Clean up the streams.
Reader.Close();
DataStream.Close();
Response.Close();
}
private void POST(string url, string postData, string tokenName)
{
string token = GetToken(tokenName);
Request = (HttpWebRequest)System.Net.WebRequest.Create(url);
Request.Method = "POST";
Request.KeepAlive = true;
Request.CookieContainer = CookieContainer;
postData = string.Format("{0}={1}&{2}", tokenName, token, postData);
byte[] byteArray = Encoding.UTF8.GetBytes(postData);
// Set the ContentType property of the WebRequest.
Request.ContentType = "application/x-www-form-urlencoded";
// Set the ContentLength property of the WebRequest.
Request.ContentLength = byteArray.Length;
// Get the request stream.
DataStream = Request.GetRequestStream();
// Write the data to the request stream.
DataStream.Write(byteArray, 0, byteArray.Length);
// Close the Stream object.
DataStream.Close();
Response = Request.GetResponse();
// Get the stream containing content returned by the server.
DataStream = Response.GetResponseStream();
// Open the stream using a StreamReader for easy access.
Reader = new StreamReader(DataStream);
// Read the content.
Page = Reader.ReadToEnd();
// Clean up the streams.
Reader.Close();
DataStream.Close();
Response.Close();
}
private void GET(string url)
{
Request = (HttpWebRequest)System.Net.WebRequest.Create(url);
Request.Method = "GET";
Request.CookieContainer = CookieContainer;
Response = Request.GetResponse();
// Get the stream containing content returned by the server.
DataStream = Response.GetResponseStream();
// Open the stream using a StreamReader for easy access.
Reader = new StreamReader(DataStream);
// Read the content.
Page = Reader.ReadToEnd();
// Clean up the streams.
Reader.Close();
DataStream.Close();
Response.Close();
}
private string GetToken(string tokenName)
{
int tokenIndex = Page.IndexOf(tokenName);
int endindex = tokenIndex + Page.Substring(tokenIndex).IndexOf(">", StringComparison.Ordinal);
int startindex = Page.Substring(0, tokenIndex).LastIndexOf("<", StringComparison.Ordinal);
string input = Page.Substring(startindex, endindex - startindex + 1);
string valuestring = "value=\"";
int tokenIndex2 = input.IndexOf(valuestring, StringComparison.Ordinal);
int endindex2 = tokenIndex2 + valuestring.Length + input.Substring(tokenIndex2 + valuestring.Length).IndexOf("\"", StringComparison.Ordinal);
int startindex2 = tokenIndex2 + valuestring.Length;
string output = input.Substring(startindex2, endindex2 - startindex2);
return output;
}
public void UpdateStatus(int[] issueIds, int statusId)
{
string tokenName = "bug_actiongroup_UP_STATUS_token";
string formUrl = GetUpdateStatusUrl(issueIds);
string formPostDataUrl = "http://mantispageurl/bug_actiongroup.php";
string formPostData = GetUpdateStatusPostData(issueIds, statusId);
GET(formUrl);
POST(formPostDataUrl, formPostData, tokenName);
}
private string GetUpdateStatusUrl(int[] issueIds)
{
string postData = "?";
foreach (var issueId in issueIds)
{
postData = string.Format("{0}&bug_arr%5B%5D={1}", postData, issueId);
}
postData = String.Format("{0}&action=UP_STATUS", postData);
return "http://mantispageurl/bug_actiongroup_page.php" + postData;
}
private string GetUpdateStatusPostData(int[] issueIds, int statusId)
{
string postData = "action=UP_STATUS";
foreach (var issueId in issueIds)
{
postData = string.Format("{0}&bug_arr%5B%5D={1}", postData, issueId);
}
postData = String.Format("{0}&status={1}&bugnote_text=", postData, statusId);
return postData;
}
I am currently having a problem.
I am trying to post my data to a PHP document but it does not get the whole value.
Somewhere in the middle it stops posting.
Does anyone know where the problem is located?
The bytearray is 7401 long. That cant be to long rigth?
My code is below:
public string RecieveData(string url, string postData = "")
{
WebRequest request = WebRequest.Create(url);
// If required by the server, set the credentials.
NetworkCredential nc = new NetworkCredential("user", "pass");
Stream dataStream;
if (postData != "")
{
// Set the Method property of the request to POST.
request.Method = "POST";
// Create POST data and convert it to a byte array.string postData = "This is a test that posts this string to a Web server.";
byte[] byteArray = Encoding.UTF8.GetBytes(postData);
// Set the ContentType property of the WebRequest.
request.ContentType = "application/x-www-form-urlencoded";
// Set the ContentLength property of the WebRequest.
request.ContentLength = byteArray.Length;
// Get the request stream.
dataStream = request.GetRequestStream();
// Write the data to the request stream.
dataStream.Write(byteArray, 0, byteArray.Length);
// Close the Stream object.
dataStream.Close();
}
request.Credentials = nc;
// Get the response.
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
// Display the status.
Console.WriteLine(response.StatusDescription);
// Get the stream containing content returned by the server.
dataStream = response.GetResponseStream();
// Open the stream using a StreamReader for easy access.
StreamReader reader = new StreamReader(dataStream);
// Read the content.
string responseFromServer = reader.ReadToEnd();
// Display the content.
Console.WriteLine(responseFromServer);
// Cleanup the streams and the response.
reader.Close();
dataStream.Close();
response.Close();
return responseFromServer;
/*
}
catch (Exception)
{
MessageBox.Show("Er is iets fout gegaan met verbinden");
return "";
}
*/
}
7401 is not too long.
My guess is that the data you are posting are not fully URL-encoded, e.g. one of the characters in the byte array causes the PhP parser to stop. Make sure you are looking at the raw data (e.g. using Wireshark).
I got a Java Servelet like
http://152.252.271.12:9999/media/servlet/RequestProcessor.do
for this I need to pass parameters in Querystring like this
http://152.252.271.12:9999/media/servlet/RequestProcessor.do?input=abc
I need to read response back.
I tried like this.
request.Method = "POST";
request.KeepAlive = false;
request.Credentials = CredentialCache.DefaultCredentials;
using (StreamWriter writer = new StreamWriter(request.GetRequestStream()))
{
writer.Write(postData);
}
// Read Response.
HttpWebResponse webresponse = (HttpWebResponse)request.GetResponse();
using (StreamReader reader = new StreamReader(webresponse.GetResponseStream(), Encoding.Default))
{
result = reader.ReadToEnd();
logtxn.LogTxn(result, "SPDCL RESPONSE");
}
webresponse.Close();
Any help appreciated..
Use WebRequest & WebResponse classes. Check http://gsraj.tripod.com/dotnet/webgetpost.html for more information.
I've had a similar problem in the past, I had to host a service in IIS that accepted a string containing a request and returned another string with a response to that request. All of the above needed to be done using HTTP POST.
The solution I found was to host a WCF service on IIS and configure it to use HTTP GET/POST protocols.
Also I remember that I needed to use a Stream class to input and output data (a simple string would not work in this case), in other words, I had a HTTP POST method in WCF that was something like:
Stream GetData(Stream data);
Take a look at this article.
You can use the WebClient class, as a client for the servlet. For example
WebClient myWebClient = new WebClient();
// Download the Web resource and save it into a data buffer.
byte[] myDataBuffer = myWebClient.DownloadData (#"http://152.252.271.12:9999/media/servlet/RequestProcessor.do?input=abc");
// Display the downloaded data.
string download = Encoding.ASCII.GetString(myDataBuffer);
Now parse the result string download and obtain the response.
Please find the below code:
public class MyWebRequest
{
private WebRequest request;
private Stream dataStream;
private string status;
public String Status
{
get
{
return status;
}
set
{
status = value;
}
}
public MyWebRequest(string url)
{
// Create a request using a URL that can receive a post.
request = WebRequest.Create(url);
}
public MyWebRequest(string url, string method): this(url)
{
if (method.Equals("GET") || method.Equals("POST"))
{
// Set the Method property of the request to POST.
request.Method = method;
}
else
{
throw new Exception("Invalid Method Type");
}
}
public MyWebRequest(string url, string method, string data): this(url, method)
{
// Create POST data and convert it to a byte array.
string postData = data;
byte[] byteArray = Encoding.UTF8.GetBytes(postData);
// Set the ContentType property of the WebRequest.
request.ContentType = "application/x-www-form-urlencoded";
// Set the ContentLength property of the WebRequest.
request.ContentLength = byteArray.Length;
// Get the request stream.
dataStream = request.GetRequestStream();
// Write the data to the request stream.
dataStream.Write(byteArray, 0, byteArray.Length);
// Close the Stream object.
dataStream.Close();
}
public string GetResponse()
{
// Get the original response.
WebResponse response = request.GetResponse();
this.Status = ((HttpWebResponse) response)
.StatusDescription;
// Get the stream containing all content returned by the requested server.
dataStream = response.GetResponseStream();
// Open the stream using a StreamReader for easy access.
StreamReader reader = new StreamReader(dataStream);
// Read the content fully up to the end.
string responseFromServer = reader.ReadToEnd();
// Clean up the streams.
reader.Close();
dataStream.Close();
response.Close();
return responseFromServer;
}
}
//create the constructor with post type and few data
MyWebRequest myRequest = new MyWebRequest("http://www.yourdomain.com", "POST", "a=value1&b=value2");
//show the response string on the console screen.
Console.WriteLine(myRequest.GetResponse());