I have the following function written in c#:
public static string FunctionName()
{
try
{
using (var httpClient = new HttpClient())
{
var uri = new Uri("https://www.website.com/api/1/auth/user");
httpClient.BaseAddress = uri;
httpClient.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36");
httpClient.DefaultRequestHeaders.Add("Cookie", "apiKey=dasdasd; auth=authcookie_dasd-c28673189043; id_chat.com=dasdasdad");
return httpClient.GetAsync(uri).Result.ToString();
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message.ToString());
}
return null;
}
where the cookie is added directly to the header. Now I was trying to split this up to add it to a CookieContainer. The source code looks like the following:
public static string FunctionName()
{
try
{
HttpClientHandler handler = new HttpClientHandler();
handler.CookieContainer = new CookieContainer();
Uri target = new Uri("https://www.website.com");
handler.CookieContainer.Add(new Cookie("apiKey", "dasdasd") { Domain = target.Host });
handler.CookieContainer.Add(new Cookie("auth", "authcookie_dasd-c28673189043") { Domain = target.Host });
handler.CookieContainer.Add(new Cookie("id_chat.com", "dasdasdad") { Domain = target.Host });
HttpClient http = new HttpClient(handler);
http.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36");
var response = http.GetAsync("https://www.website.com/api/1/auth/user");
Console.WriteLine(response.Result.ToString());
if (response.Result.StatusCode == HttpStatusCode.OK)
{
var json = JsonConvert.DeserializeObject<LoginResponse>(response.Result.Content.ReadAsStringAsync().Result);
return json.id;
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message.ToString());
}
return null;
}
The first function would return the status "OK".
The second function would return the status "Unauthorized".
What is causing this problem? Did I set up the CookieContainer wrong?
Related
I have tried with the following code.The error received is "The underlying connection was closed: An unexpected error occurred on a receive".
System OS: Windows 7 Home Basic SP1
.net Version: 4.5.2
using System;
using System.Net;
using System.Net.Security;
namespace wc_downloadfile
{
class Program
{
public static void Main(string[] args)
{
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
ServicePointManager.ServerCertificateValidationCallback += (sender, cert, chain, sslPolicyErrors) => true;
using(WebClient wc = new WebClient())
{
string src = "https://nseindia.com/content/indices/ind_nifty50list.csv";
string dest = #"C:\temp\N50.csv";
try
{
var ua = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36";
wc.Headers.Add(HttpRequestHeader.UserAgent, ua);
wc.Headers.Add(HttpRequestHeader.Accept, "*/*");
wc.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip, deflate");
wc.DownloadFile(src, dest);
}
catch (Exception e)
{
Console.WriteLine("Unable to download file --- " + e.Message);
}
}
Console.ReadKey(true);
}
}
}
Modified the user agent value and the problem got resolved. Following is the working code.
using System;
using System.Net;
using System.Net.Security;
namespace wc_downloadfile
{
class Program
{
public static void Main(string[] args)
{
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
ServicePointManager.ServerCertificateValidationCallback += (sender, cert, chain, sslPolicyErrors) => true;
using(WebClient wc = new WebClient())
{
string src = "https://nseindia.com/content/indices/ind_nifty50list.csv";
string dest = #"C:\temp\N50.csv";
try
{
//Removed Chrome/61.0.3163.100
var ua = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36";
wc.Headers.Add(HttpRequestHeader.UserAgent, ua);
wc.Headers.Add(HttpRequestHeader.Accept, "*/*");
wc.DownloadFile(src, dest);
}
catch (Exception e)
{
Console.WriteLine("Unable to download file --- " + e.Message);
}
}
Console.ReadKey(true);
}
}
}
WebClient client = new WebClient();
client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)");
Stream stream = client.OpenRead("https://nseindia.com/content/indices/ind_nifty50list.csv");
StreamReader reader = new StreamReader(stream);
String content = reader.ReadToEnd();
I cant force to work HttpClient Post request...
or rather, I can't understand what I'm doing wrong..
My code below and this's sketch
#region Method 2
HttpClient h = new HttpClient();
var values = new Dictionary<string, string>
{
//{ "csrf_token" , "aacc347b22df2a7b3a20c9674ba59cf4279c8d16:1552967983844" },
//{ "track_id", "501c86f19b745703e9ca54991e90dc3a87" },
//{ "password", "Oguhokuu13" }
};
CookieContainer container = new CookieContainer();
h.DefaultRequestHeaders.Add("Accept", "application/json, text/javascript, */*; q=0.01");
h.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate, br");
h.DefaultRequestHeaders.Add("Accept-Language", "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7");
h.DefaultRequestHeaders.Add("Connection", "keep-alive");
h.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/x-www-form-urlencoded")); // Content-Type
h.DefaultRequestHeaders.AcceptCharset.Add(new StringWithQualityHeaderValue("UTF-8"));
h.DefaultRequestHeaders.Add("Origin", "https://passport.yandex.ru");
h.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36");
// h.DefaultRequestHeaders.Add("Referer", "https://passport.yandex.ru/auth/welcome?retpath=https%3A%2F%2Ftaxi-cabinet.mobile.yandex.ru%2Fnews%2F");
h.DefaultRequestHeaders.Add("Host", "passport.yandex.ru");
h.DefaultRequestHeaders.Add("X-Requested-With", "XMLHttpRequest");
var content = new FormUrlEncodedContent(values);
var response = h.PostAsync("https://passport.yandex.ru/auth/add", content);
response.Wait();
var responseString = response.Result.Content.ReadAsStringAsync().Result;
textBox1.Text = responseString.ToString();
#endregion
Then I get response:
\u001f?\b\0\0\0\0\0\0\u0003sN???/Q\b?\u000f\u000eQ?O,-??OLI?\u0002\0d(,h\u0016\0\0\0"
Idk what I'm doing wrong
private void download_nse()
{
string source_location = "https://www.nseindia.com/content/historical/DERIVATIVES/2015/DEC/fo23DEC2015bhav.csv.zip";
Uri uu = new Uri(source_location);
using (WebClient fileReader = new WebClient())
{
try
{
var ua = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36";
fileReader.Headers.Add(HttpRequestHeader.UserAgent, ua);
fileReader.Headers["Accept"] = "/";
fileReader.DownloadFile(uu, #"d:\nse\notworking.zip");
fileReader.Dispose();
}
catch
{
}
}
source_location = "https://www.nseindia.com/content/historical/DERIVATIVES/2016/JAN/fo05JAN2016bhav.csv.zip";
using (WebClient fileReader = new WebClient())
{
try
{
var ua = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36";
fileReader.Headers.Add(HttpRequestHeader.UserAgent, ua);
fileReader.Headers["Accept"] = "/";
fileReader.DownloadFile(source_location, #"d:\nse\working.zip");
}
catch
{
}
}
}
I've checked the url you gave and see that when you select the "Date(DD-MM-YYYY)" and "Select Report" drop-down options and click on "Get Data", there is a GET request send to https://www.nseindia.com/ArchieveSearch with three parameter like this: "?h_filetype=fobhav&date=02-04-2018§ion=FO". And this GET request returns:
<p class="archive_title">F&O - Bhavcopy for 02-04-2018 </p><br>
<br><table cellpadding=5>
<tr>
<td class=t0><a href=/content/historical/DERIVATIVES/2018/APR/fo02APR2018bhav.csv.zip target=new>fo02APR2018bhav.csv.zip</a></td></tr>
<tr>
</table>
</td></tr>
</table>
which contains the link to download file. so when you send the POST request without the GET it fails.
You can send a GET request first and then send POST with the returned link like this:
string source_location2 = "https://www.nseindia.com/ArchieveSearch";
Uri uu2 = new Uri(source_location2);
using (WebClient fileReader = new WebClient())
{
try
{
var ua = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36";
fileReader.Headers.Add(HttpRequestHeader.UserAgent, ua);
fileReader.Headers["Accept"] = "/";
fileReader.QueryString.Add("h_filetype", "fobhav");
fileReader.QueryString.Add("date", "02-04-2018");
fileReader.QueryString.Add("section", "FO");
var response = fileReader.DownloadString(uu2);
//using Html Agility Pack to parse the response and get the download link. you need to add this package through nuget package manager to project for this code to work.
var htmlDoc = new HtmlDocument();
htmlDoc.LoadHtml(response);
var fileLink = htmlDoc.DocumentNode.SelectSingleNode("//table//tr//td//a").Attributes["href"].Value;
//now sending the actual file request...
var fileReader2 = new WebClient();
fileReader2.Headers.Add(HttpRequestHeader.UserAgent, ua);
fileReader2.Headers["Accept"] = "/";
fileReader2.DownloadFile(new Uri("https://www.nseindia.com" + fileLink), #"d:\notworking.zip");
fileReader2.Dispose();
}
catch(Exception e)
{
throw e;
}
}
I try to get some data from a web, but always get 401 Unauthorized error. I guess this is caused by my request header setting is wrong.
Could we can set Accept, ContentType by using TryAddWithoutValidation?
How can I set the correct request header in HttpClient?
And I try to set the header like as following:
public static async Task<ResultInfo> Get(string url, string cookie = null)
{
url =$"https://www.zhihu.com/api/v4/questions/46508954/answers?sort_by=default&include=data%5B%2A%5D.is_normal%2Cis_sticky%2Ccollapsed_by%2Csuggest_edit%2Ccomment_count%2Ccan_comment%2Ccontent%2Ceditable_content%2Cvoteup_count%2Creshipment_settings%2Ccomment_permission%2Cmark_infos%2Ccreated_time%2Cupdated_time%2Crelationship.is_authorized%2Cis_author%2Cvoting%2Cis_thanked%2Cis_nothelp%2Cupvoted_followees%3Bdata%5B%2A%5D.author.badge%5B%3F%28type%3Dbest_answerer%29%5D.topics&limit=20&offset=20";
; var result = new ResultInfo()
{
IsSuccessStatusCode = false
};
HttpClientHandler handler = new HttpClientHandler()
{
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
};
using (var client = new HttpClient(handler))
{
try
{
cookie = "_zap=bb597634-86d0-4636-a516-89e76b4b5335; q_c1=b2e3af8dde034209a6948fcd9fd0e70f|1515131469000|1509512040000; d_c0=\"AJBrsuC5AA2PTqu9qOLgytIZlpXzmzgRgcE=|1516167026\"; _xsrf=ec351d2e-f0b2-4b2e-9eff-a3262a9faf10; capsion_ticket=\"2|1:0|10:1516261057|14:capsion_ticket|44:NzU1ZGRjYTA3Y2U3NGVjZWE4MWIzMzllNjA1NjMxZGQ=|b73bb13fe6a623db85478f5eb46ed51906674b89f29e4ef1a067fce6dcd90f54\"; z_c0=\"2|1:0|10:1516261068|4:z_c0|92:Mi4xMzMwVEFBQUFBQUFBa0d1eTRMa0FEU1lBQUFCZ0FsVk56SnhOV3dETFF5VGlla1ZDUllwcUVMRVhMWk4wV1duOVJn|f2fcc1750b966a19dd202ddc2eb42dca9dc9f573421bc3a8675a072e603ca897\"; __utma=51854390.1320208938.1517560249.1517560249.1517560249.1; __utmb=51854390.0.10.1517560249; __utmc=51854390; __utmz=51854390.1517560249.1.1.utmcsr=zhihu.com|utmccn=(referral)|utmcmd=referral|utmcct=/question/38824940; __utmv=51854390.100-1|2=registration_date=20130816=1^3=entry_date=20130816=1";
var replace = cookie.Replace("\"", "");
client.DefaultRequestHeaders.TryAddWithoutValidation("Accept", "*/*");
client.DefaultRequestHeaders.TryAddWithoutValidation("ContentType", "application/x-www-form-urlencoded");
client.DefaultRequestHeaders.TryAddWithoutValidation("UserAgent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.108 Safari/537.36");
client.DefaultRequestHeaders.TryAddWithoutValidation("Cookie", replace);
client.DefaultRequestHeaders.TryAddWithoutValidation("accept-encoding", "gzip/deflate");
//client.DefaultRequestHeaders.TryAddWithoutValidation("accept-language", "en-US,en;q=0.9");
client.DefaultRequestHeaders.TryAddWithoutValidation("AllowAutoRedirect","false");
if (!string.IsNullOrEmpty(url))
{
var data = await client.GetAsync(url);
if (data.IsSuccessStatusCode)
{
var content = await data.Content.ReadAsStringAsync();
result.IsSuccessStatusCode = true;
result.Data = await data.Content.ReadAsStringAsync();
}
else
{
result.Message = data.StatusCode.ToString();
}
}
}
catch (Exception e)
{
result.Message = e.Message;
throw;
}
}
return result;
}
I have a problem using the response of HttpRequest() i get the response but just the html not the headers and the key that i am searching is on the header so this is my code
HttpRequest rq = new HttpRequest();
rq.Cookies = new CookieDictionary();
rq.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36";
rq.AllowAutoRedirect = true;
rq.IgnoreProtocolErrors = true;
rq.ConnectTimeout = TimeOut;
rq.KeepAlive = true;
var str = rq.Get("url").ToString();
if(str.Contains("404")){
}
i hope you can help me
I Found the answer thanks for your help
var req = rq.Get("url");
if(req.StatusCode.ToString().Contains("NotFound") ){
}