I have to scrape a table from a secure site and I'm having trouble logging in to the page and retrieving the authentication token and any other associated cookies. Am I doing something wrong here?
public NameValueCollection LoginToDatrose()
{
var loginUriBuilder = new UriBuilder();
loginUriBuilder.Host = DatroseHostName;
loginUriBuilder.Path = BuildURIPath(DatroseBasePath, LOGIN_PAGE);
loginUriBuilder.Scheme = "https";
var boundary = Guid.NewGuid().ToString();
var postData = new NameValueCollection();
postData.Add("LoginName", DatroseUserName);
postData.Add("Password", DatrosePassword);
var data = Encoding.ASCII.GetBytes(postData.ToQueryString(false));
var request = WebRequest.Create(loginUriBuilder.Uri) as HttpWebRequest;
request.Method = "POST";
request.ContentType = "application/x-www-form-urlencoded";
request.ContentLength = data.Length;
using (var d = request.GetRequestStream())
{
d.Write(data, 0, data.Length);
}
var response = request.GetResponse() as HttpWebResponse;
var responseCookies = new NameValueCollection();
foreach (var nvp in response.Cookies.OfType<Cookie>())
{
responseCookies.Add(nvp.Name, nvp.Value);
}
//using (var responseData = response.GetResponseStream())
//using (var responseReader = new StreamReader(responseData))
//{
// var theResponse = responseReader.ReadToEnd();
// Debug.WriteLine(theResponse);
//}
return responseCookies;
}
I get no values in the return object. It does not fail. The value of theResponse (when not commented out) seems to be the HTML of the login page.
Any assistance would be greatly appreciated.
OK, the problem here seems related to the 302 redirect that would occur after the credentials were passed. The HttpWebRequest would automatically follow the 302.
Ultimately, I ended up doing things a little differently. First, I subclassed the WebClient class as follows:
public class CookiesAwareWebClient : WebClient
{
private CookieContainer outboundCookies = new CookieContainer();
private CookieCollection inboundCookies = new CookieCollection();
public CookieContainer OutboundCookies
{
get
{
return outboundCookies;
}
}
public CookieCollection InboundCookies
{
get
{
return inboundCookies;
}
}
public bool IgnoreRedirects { get; set; }
protected override WebRequest GetWebRequest(Uri address)
{
WebRequest request = base.GetWebRequest(address);
if (request is HttpWebRequest)
{
(request as HttpWebRequest).CookieContainer = outboundCookies;
(request as HttpWebRequest).AllowAutoRedirect = !IgnoreRedirects;
}
return request;
}
protected override WebResponse GetWebResponse(WebRequest request)
{
WebResponse response = base.GetWebResponse(request);
if (response is HttpWebResponse)
{
inboundCookies = (response as HttpWebResponse).Cookies ?? inboundCookies;
}
return response;
}
}
This allowed me to use a WebClient class that was cookies-aware as well as one that I could control the redirect. Then I rewrote my code for logging in as follows:
public NameValueCollection LoginToDatrose()
{
var loginUriBuilder = new UriBuilder();
loginUriBuilder.Host = DatroseHostName;
loginUriBuilder.Path = BuildURIPath(DatroseBasePath, LOGIN_PAGE);
loginUriBuilder.Scheme = "https";
var postData = new NameValueCollection();
postData.Add("LoginName", DatroseUserName);
postData.Add("Password", DatrosePassword);
var responseCookies = new NameValueCollection();
using (var client = new CookiesAwareWebClient())
{
client.IgnoreRedirects = true;
var clientResponse = client.UploadValues(loginUriBuilder.Uri, "POST", postData);
foreach (var nvp in client.InboundCookies.OfType<Cookie>())
{
responseCookies.Add(nvp.Name, nvp.Value);
}
}
return responseCookies;
}
...and everything worked swimmingly.
Related
I'm trying to make web requests programmatically in ASP.NET, using the POST method.
I'd like to send POST parameters with the web request as well. Something like this:
LoginData obj = new LoginData();
obj.OSVersion = deviceInformation.OperatingSystem;
obj.DeviceModel = deviceInformation.FriendlyName;
string URI = "https://XXXXXXXXX.azure-mobile.net/user/logsuserin";
HttpWebRequest GETRequest = (HttpWebRequest)HttpWebRequest.Create(new Uri(URI, UriKind.RelativeOrAbsolute));
GETRequest.Method = "POST";
GETRequest.ContentType = "application/x-www-form-urlencoded";
GETRequest.Headers["applicationKey"] = "UFakeKkrayuAeVnoVAcjY54545455544";
//GETRequest.Parameters.add(obj);
Obviously, the commented line does not work. How do I achieve this?
How to get a response by sending my obj as params?
Thanks in advance,
Hemanth.
You need to use theGetRequestStream() method belonging to the HttpWebRequest
void Main()
{
LoginData obj = new LoginData
{
Username = "foo",
Password = "Bar"
};
byte[] objBytes = Encoding.UTF8.GetBytes(obj.ToString());
// obj.OSVersion = deviceInformation.OperatingSystem;
// obj.DeviceModel = deviceInformation.FriendlyName;
string URI = "https://XXXXXXXXX.azure-mobile.net/user/logsuserin";
HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(new Uri(URI, UriKind.RelativeOrAbsolute));
request.Method = "POST";
request.ContentType = "application/x-www-form-urlencoded";
request.Headers["applicationKey"] = "UFakeKkrayuAeVnoVAcjY54545455544";
request.ContentLength = objBytes.Length;
using (Stream stream = request.GetRequestStream())
{
stream.Write(objBytes, 0, objBytes.Length);
}
using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
using (Stream stream = response.GetResponseStream())
using (StreamReader reader = new StreamReader(stream))
{
Console.WriteLine(reader.ReadToEnd());
}
}
public class LoginData
{
public string Username { get; set; }
public string Password { get; set; }
public string OSVersion { get; set; }
public string DeviceModel { get; set; }
public override string ToString()
{
var temp = this.GetType()
.GetProperties()
.Select(p => $"{p.Name}={HttpUtility.UrlEncode(p.GetValue(this).ToString())}");
return string.Join("&", temp);
}
}
If you want to use HttpClient:
using (var client = new HttpClient())
{
var request = new HttpRequestMessage(HttpMethod.Post, "https://XXXXXXXXX.azure-mobile.net/user/logsuserin");
request.Headers.Add("applikationKey", "UFakeKkrayuAeVnoVAcjY54545455544");
request.Content = new FormUrlEncodedContent(new[]
{
new KeyValuePair<string, string>("OSVersion", deviceInformation.OperatingSystem),
new KeyValuePair<string, string>("DeviceModel", deviceInformation.FriendlyName),
});
var response = client.SendAsync(request).GetAwaiter().GetResult();
}
You can dynamically generate a FORM with "NameValueCollection". Using "NameValueCollection" you can add number of objects to be posted as -
NameValueCollection FormFields = new NameValueCollection();
FormFields.Add("abc", obj1);
FormFields.Add("xyz", obj2);
Response.Clear();
Response.Write("<html><head>");
Response.Write(string.Format("</head><body onload=\"document.{0}.submit()\">", FormName));
Response.Write(string.Format("<form name=\"{0}\" method=\"{1}\" action=\"{2}\" >", FormName, Method, Url));
for (int i = 0; i < FormFields.Keys.Count; i++)
{
Response.Write(string.Format("<input name=\"{0}\" type=\"hidden\" value=\"{1}\">", FormFields.Keys[i], FormFields[FormFields.Keys[i]]));
}
Response.Write("</form>");
Response.Write("</body></html>");
Response.End();
OnLoad() of this form you can POST to desired URL.
I've written below C# code to login to JIRA Rest API:
var url = new Uri("http://localhost:8090/rest/auth/latest/session?os_username=tempusername&os_password=temppwd");
var request = WebRequest.Create(url) as HttpWebRequest;
if (null == request)
{
return "";
}
request.Method = "POST";
request.ContentType = "application/json";
request.ContentLength = 200;
request.KeepAlive = false;
using (var response = request.GetResponse() as HttpWebResponse)
{
}
When I execute this, application just goes on running without returning any response. Please suggest if this is the right way of calling JIRA Login using REST API
For basic authentication you need to send in the username and password in a base64-encoding. Guidelines can be found in the API examples on atlassians developer page:
https://developer.atlassian.com/display/JIRADEV/JIRA+REST+API+Example+-+Basic+Authentication
, if you are doing it in C# you need to send the encoded data in the header in the following format:
"Authorization: Basic [ENCODED CREDENTIALS]"
Here is a simple example:
public enum JiraResource
{
project
}
protected string RunQuery(
JiraResource resource,
string argument = null,
string data = null,
string method = "GET")
{
string url = string.Format("{0}{1}/", m_BaseUrl, resource.ToString());
if (argument != null)
{
url = string.Format("{0}{1}/", url, argument);
}
HttpWebRequest request = WebRequest.Create(url) as HttpWebRequest;
request.ContentType = "application/json";
request.Method = method;
if (data != null)
{
using (StreamWriter writer = new StreamWriter(request.GetRequestStream()))
{
writer.Write(data);
}
}
string base64Credentials = GetEncodedCredentials();
request.Headers.Add("Authorization", "Basic " + base64Credentials);
HttpWebResponse response = request.GetResponse() as HttpWebResponse;
string result = string.Empty;
using (StreamReader reader = new StreamReader(response.GetResponseStream()))
{
result = reader.ReadToEnd();
}
return result;
}
private string GetEncodedCredentials()
{
string mergedCredentials = string.Format("{0}:{1}", m_Username, m_Password);
byte[] byteCredentials = UTF8Encoding.UTF8.GetBytes(mergedCredentials);
return Convert.ToBase64String(byteCredentials);
}
(JiraResource is just an enum I use to decide which part of the API to use)
I hope this will help!
Here is a simpler solution which works as required:
var mergedCredentials = string.Format("{0}:{1}", username, password);
var byteCredentials = Encoding.UTF8.GetBytes(mergedCredentials);
var encodedCredentials = Convert.ToBase64String(byteCredentials);
using (WebClient webClient = new WebClient())
{
webClient.Headers.Set("Authorization", "Basic " + encodedCredentials);
return webClient.DownloadString(url);
}
If you don't want to encode your credentials in every request here is how to do it using cookies.
When requesting the cookie you don't need to add any authorization on the headers. This method will accept a JSON string with the user name and password and the URL. It will return the cookie values.
public async Task<JiraCookie> GetCookieAsync(string myJsonUserNamePassword, string JiraCookieEndpointUrl)
{
using (var client = new HttpClient())
{
var response = await client.PostAsync(
JiraCookieEndpointUrl,
new StringContent(myJsonUserNamePassword, Encoding.UTF8, "application/json"));
var json = response.Content.ReadAsStringAsync().Result;
var jiraCookie= JsonConvert.DeserializeObject<JiraCookie>(json);
return jArr;
}
}
public class JiraCookie
{
public Session session { get; set; }
}
public class Session
{
public string name { get; set; }
public string value { get; set; }
}
When I call it using url: http://[baseJiraUrl]/rest/auth/1/session it returns the following JSON response:
{
"session" : -{
"name" : JSESSIONID,
"value" : cookieValue
}
Keep in mind the URL above is valid in the version of JIRA I'm using and may vary depending on which version you're using. Read the JIRA API documentation for the correct URL for the version you are using. I'm using the following:
https://docs.atlassian.com/software/jira/docs/api/REST/7.6.1/#auth/1/session
Remember you'll have to store your cookie and use it on every subsequent request.
Check out this answer on how add cookies to your HttpClient request: How do I set a cookie on HttpClient's HttpRequestMessage.
Once you're done with the cookie (logging out) simply send a delete http request with the same URL as the post.
I tweaked the RunQuery code so that it will run today (Apr 2018). The encrypt/decrypt referenced below is from the following link (I converted it to an extension method and threw values into environment).
https://stackoverflow.com/questions/10168240/encrypting-decrypting-a-string-in-c-sharp
I successfully execute the code from LinqPad - thus the Dump() command after RunQuery
private string _baseUrl = "https://xxxxxx.atlassian.net";
private string _username = "YourLogin";
void Main()
{
RunQuery(JiraResource.project).JsonToXml().Dump();
}
public enum JiraResource { project }
private const string restApiVersion = "/rest/api/2/";
protected string RunQuery( JiraResource resource, string argument = null, string data = null, string method = "GET")
{
string url = $"{_baseUrl}{restApiVersion}{resource}";
if (argument != null) url = $"{url}{argument}/";
var request = WebRequest.Create(url) as HttpWebRequest;
request.ContentType = "application/json";
request.Method = method;
if (data != null)
{
using (StreamWriter writer = new StreamWriter(request.GetRequestStream()))
{
writer.Write(data);
}
}
string base64Credentials = GetEncodedCredentials();
request.Headers.Add("Authorization", "Basic " + base64Credentials);
var response = request.GetResponse() as HttpWebResponse;
string result = string.Empty;
using (StreamReader reader = new StreamReader(response.GetResponseStream()))
{
result = reader.ReadToEnd();
}
return result;
}
private string GetEncodedCredentials()
{
var encryptedPassword = Environment.GetEnvironmentVariable("PassEncrypted");
var encryptionSalt = Environment.GetEnvironmentVariable("PassSalt");
var password = encryptedPassword.Decrypt(encryptionSalt);
var mergedCredentials = $"{_username}:{password}";
var byteCredentials = UTF8Encoding.UTF8.GetBytes(mergedCredentials);
return Convert.ToBase64String(byteCredentials);
}
public static class MyExtensions
{
public static XElement JsonToXml(this string jsonData, bool isAddingHeader = true)
{
var data = isAddingHeader
? "{\"record\":" + jsonData + "}"
: jsonData;
data = data // Complains if xml element name starts numeric
.Replace("16x16", "n16x16")
.Replace("24x24", "n24x24")
.Replace("32x32", "n32x32")
.Replace("48x48", "n48x48");
var result = JsonConvert.DeserializeXmlNode(data, "data");
var xmlResult = XElement.Parse(result.OuterXml);
return xmlResult;
}
}
For posting multipart content in Rest I use Tiny.RestClient.
var client = new TinyRestClient(new HttpClient(), "http://localhost:8090");
var strResult = await client.PostRequest("rest/auth/latest/session).
WithBasicAuthentication("username", "password")
ExecuteAsStringAsync();
static void Main(string[] args)
{
using (WebClient wc = new WebClient())
{
wc.Headers.Add("Authorization", "Basic " + GetEncodedCredentials());
string tasks = wc.DownloadString("yourjiraurl/search?jql=task=bug");
var taskdetails = JsonConvert.DeserializeObject<TaskDetails>(tasks);
}
}
static string GetEncodedCredentials()
{
string mergedCredentials = string.Format("{0}:{1}", "UserName", "Password");
byte[] byteCredentials = UTF8Encoding.UTF8.GetBytes(mergedCredentials);
return Convert.ToBase64String(byteCredentials);
}
Actually I have something like this:
private void createHttpRequest()
{
System.Uri myUri = new System.Uri("..url..");
HttpWebRequest myHttpRequest = (HttpWebRequest)HttpWebRequest.Create(myUri);
myHttpRequest.Method = "POST";
myHttpRequest.ContentType = "application/x-www-form-urlencoded";
myHttpRequest.BeginGetRequestStream(new AsyncCallback(GetRequestStreamCallback), myHttpRequest);
}
void GetRequestStreamCallback(IAsyncResult callbackResult)
{
HttpWebRequest myRequest = (HttpWebRequest)callbackResult.AsyncState;
// End the stream request operation
Stream postStream = myRequest.EndGetRequestStream(callbackResult);
string hash = HashHelper.createStringHash("123", "TEST", "0216");
// Create the post data
byte[] byteArray = createByteArrayFromHash(hash);
// Add the post data to the web request
postStream.Write(byteArray, 0, byteArray.Length);
postStream.Close();
// Start the web request
myRequest.BeginGetResponse(new AsyncCallback(GetResponsetStreamCallback), myRequest);
}
void GetResponsetStreamCallback(IAsyncResult callbackResult)
{
HttpWebRequest request = (HttpWebRequest)callbackResult.AsyncState;
HttpWebResponse response = (HttpWebResponse)request.EndGetResponse(callbackResult);
using (StreamReader httpWebStreamReader = new StreamReader(response.GetResponseStream()))
{
string result = httpWebStreamReader.ReadToEnd();
ApiResponse apiResponse = (ApiResponse)JsonConvert.DeserializeObject<ApiResponse>(result);
}
}
It's good, it's working but now I must use these methods in every page and just change method createByteArrayFromHash which creates request. What if I want to create helper class that can help me to do this in something about 3 lines of code in page. How would you do that? I was thinking about this way but how to add request before response? Or would you do it another way? Thanks
Yeah, it's better to use async and await. Here is an example of such a wrapper:
public async Task<string> SendRequestGetResponse(string postData, CookieContainer cookiesContainer = null)
{
var postRequest = (HttpWebRequest)WebRequest.Create(Constants.WebServiceUrl);
postRequest.ContentType = "Your content-type";
postRequest.Method = "POST";
postRequest.CookieContainer = new CookieContainer();
postRequest.CookieContainer = App.Session.Cookies;
using (var requestStream = await postRequest.GetRequestStreamAsync())
{
byte[] postDataArray = Encoding.UTF8.GetBytes(postData);
await requestStream.WriteAsync(postDataArray, 0, postDataArray.Length);
}
var postResponse = await postRequest.GetResponseAsync() as HttpWebResponse;
if (postResponse != null)
{
var postResponseStream = postResponse.GetResponseStream();
var postStreamReader = new StreamReader(postResponseStream);
// Can use cookies if you need
if (cookiesContainer == null)
{
if (!string.IsNullOrEmpty(postResponse.Headers["YourCookieHere"]))
{
var cookiesCollection = postResponse.Cookies;
// App.Session is a global object to store cookies and etc.
App.Session.Cookies.Add(new Uri(Constants.WebServiceUrl), cookiesCollection);
}
}
string response = await postStreamReader.ReadToEndAsync();
return response;
}
return null;
}
You can modify it as you wish
I am encountering a problem getting the access_token in client application using oauth.
The returned response has empty body though in API I can see the response is not empty.
tokenresponse = {
"access_token":"[ACCESSTOKENVALUE]",
"token_type":"bearer",
"expires_in":"1200",
"refresh_token":"[REFRESHTOKENVALUE]",
"scope":"[SCOPEVALUE]"
}
The method from API that returns the token http://api.sample.com/OAuth/Token:
public ActionResult Token()
{
OutgoingWebResponse response =
this.AuthorizationServer.HandleTokenRequest(this.Request);
string tokenresponse = string.Format("Token({0})", response!=null?response.Body:""));
return response.AsActionResult();
}
The client method that requests the token is:
public string GetAuthorizationToken(string code)
{
string Url = ServerPath + "OAuth/Token";
string redirect_uri_encode = UrlEncode(ClientPath);
string param = string.Format("code={0}&client_id={1}&client_secret={2}&redirect_uri={3}&grant_type=authorization_code",code, ClientId, ClientSecret, redirect_uri_encode);
HttpWebRequest request = HttpWebRequest.Create(Url) as HttpWebRequest;
string result = null;
request.Method = "POST";
request.KeepAlive = true;
request.ContentType = "application/x-www-form-urlencoded";
request.Timeout = 10000;
request.Headers.Remove(HttpRequestHeader.Cookie);
var bs = Encoding.UTF8.GetBytes(param);
using (Stream reqStream = request.GetRequestStream())
{
reqStream.Write(bs, 0, bs.Length);
}
using (WebResponse response = request.GetResponse())
{
var sr = new StreamReader(response.GetResponseStream());
result = sr.ReadToEnd();
sr.Close();
}
if (!string.IsNullOrEmpty(result))
{
TokenData tokendata = JsonConvert.DeserializeObject<TokenData>(result);
return UpdateAuthorizotionFromToken(tokendata);
}
return null;
}
The result variable is empty.
Please let me know if you have any idea what could cause this. Initially I assumed is because of the cookies so I tried to remove them from request.
Thanks in advance.
Dear just create webclient using following code and you will get json info in tokeninfo.I used it and simply its working perfect.
WebClient client = new WebClient();
string postData = "client_id=" + ""
+ "&client_secret=" + ""
+ "&grant_type=password&username=" + "" //your username
+ "&password=" + "";//your password :)
string soundCloudTokenRes = "https://api.soundcloud.com/oauth2/token";
string tokenInfo = client.UploadString(soundCloudTokenRes, postData);
You can then use substring that contains only token from tokeninfo.
To upload tracks on sound cloud.
private void TestSoundCloudupload()
{
System.Net.ServicePointManager.Expect100Continue = false;
var request = WebRequest.Create("https://api.soundcloud.com/tracks") as HttpWebRequest;
//some default headers
request.Accept = "*/*";
request.Headers.Add("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.3");
request.Headers.Add("Accept-Encoding", "gzip,deflate,sdch");
request.Headers.Add("Accept-Language", "en-US,en;q=0.8,ru;q=0.6");
//file array
var files = new UploadFile[] { new UploadFile(Server.MapPath("Downloads//0.mp3"), "track[asset_data]", "application/octet-stream") };
//other form data
var form = new NameValueCollection();
form.Add("track[title]", "Some title");
form.Add("track[sharing]", "public");
form.Add("oauth_token", "");
form.Add("format", "json");
form.Add("Filename", "0.mp3");
form.Add("Upload", "Submit Query");
try
{
using (var response = HttpUploadHelper.Upload(request, files, form))
{
using (var reader = new StreamReader(response.GetResponseStream()))
{
Response.Write(reader.ReadToEnd());
}
}
}
catch (Exception ex)
{
Response.Write(ex.ToString());
}
}
I am trying to unit test some code, and I need to to replace this:
HttpWebRequest httpWebRequest = (HttpWebRequest)WebRequest.Create( uri );
httpWebRequest.CookieContainer = new CookieContainer();
with
WebRequest webRequest = WebRequest.Create( uri );
webRequest.CookieContainer = new CookieContainer();
Basically, how do I get cookies into the request without using a HttpWebRequest?
Based on your comments, you might consider writing an extension method:
public static bool TryAddCookie(this WebRequest webRequest, Cookie cookie)
{
HttpWebRequest httpRequest = webRequest as HttpWebRequest;
if (httpRequest == null)
{
return false;
}
if (httpRequest.CookieContainer == null)
{
httpRequest.CookieContainer = new CookieContainer();
}
httpRequest.CookieContainer.Add(cookie);
return true;
}
Then you can have code like:
WebRequest webRequest = WebRequest.Create( uri );
webRequest.TryAddCookie(new Cookie("someName","someValue"));
Try with something like this:
HttpWebRequest request = (HttpWebRequest)WebRequest.Create("http://www.contoso.com/default.html");
request.CookieContainer = new CookieContainer();
request.CookieContainer.Add(new Cookie("ConstoCookie", "Chocolate Flavour"));
WebRequest is an abstract class that does not have a CookieContainer property. In addition you can't use the Headers collection (not implemented exception) so any attempt like webRequest.Headers.Add("Cookie", "...") will fail.
Sorry, but you have no chance to use cookies with WebRequest.
Stick on HttpWebRequest and add/edit as many cookies you like using its Headers collection!
dlev's answer ended up working, but I had problems implementing the solution ("The parameter '{0}' cannot be an empty string."), so I decided to write the full code in case anybody else has similar problems.
My goal was to get the html as a string, but I needed to add the cookies to the web request. This is the function that downloads the string using the cookies:
public static string DownloadString(string url, Encoding encoding, IDictionary<string, string> cookieNameValues)
{
using (var webClient = new WebClient())
{
var uri = new Uri(url);
var webRequest = WebRequest.Create(uri);
foreach(var nameValue in cookieNameValues)
{
webRequest.TryAddCookie(new Cookie(nameValue.Key, nameValue.Value, "/", uri.Host));
}
var response = webRequest.GetResponse();
var receiveStream = response.GetResponseStream();
var readStream = new StreamReader(receiveStream, encoding);
var htmlCode = readStream.ReadToEnd();
return htmlCode;
}
}
We are using the code from dlev's answer:
public static bool TryAddCookie(this WebRequest webRequest, Cookie cookie)
{
HttpWebRequest httpRequest = webRequest as HttpWebRequest;
if (httpRequest == null)
{
return false;
}
if (httpRequest.CookieContainer == null)
{
httpRequest.CookieContainer = new CookieContainer();
}
httpRequest.CookieContainer.Add(cookie);
return true;
}
This is how you use the full code:
var cookieNameValues = new Dictionary<string, string>();
cookieNameValues.Add("varName", "varValue");
var htmlResult = DownloadString(url, Encoding.UTF8, cookieNameValues);