I have tried with the following code.The error received is "The underlying connection was closed: An unexpected error occurred on a receive".
System OS: Windows 7 Home Basic SP1
.net Version: 4.5.2
using System;
using System.Net;
using System.Net.Security;
namespace wc_downloadfile
{
class Program
{
public static void Main(string[] args)
{
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
ServicePointManager.ServerCertificateValidationCallback += (sender, cert, chain, sslPolicyErrors) => true;
using(WebClient wc = new WebClient())
{
string src = "https://nseindia.com/content/indices/ind_nifty50list.csv";
string dest = #"C:\temp\N50.csv";
try
{
var ua = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36";
wc.Headers.Add(HttpRequestHeader.UserAgent, ua);
wc.Headers.Add(HttpRequestHeader.Accept, "*/*");
wc.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip, deflate");
wc.DownloadFile(src, dest);
}
catch (Exception e)
{
Console.WriteLine("Unable to download file --- " + e.Message);
}
}
Console.ReadKey(true);
}
}
}
Modified the user agent value and the problem got resolved. Following is the working code.
using System;
using System.Net;
using System.Net.Security;
namespace wc_downloadfile
{
class Program
{
public static void Main(string[] args)
{
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
ServicePointManager.ServerCertificateValidationCallback += (sender, cert, chain, sslPolicyErrors) => true;
using(WebClient wc = new WebClient())
{
string src = "https://nseindia.com/content/indices/ind_nifty50list.csv";
string dest = #"C:\temp\N50.csv";
try
{
//Removed Chrome/61.0.3163.100
var ua = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36";
wc.Headers.Add(HttpRequestHeader.UserAgent, ua);
wc.Headers.Add(HttpRequestHeader.Accept, "*/*");
wc.DownloadFile(src, dest);
}
catch (Exception e)
{
Console.WriteLine("Unable to download file --- " + e.Message);
}
}
Console.ReadKey(true);
}
}
}
WebClient client = new WebClient();
client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)");
Stream stream = client.OpenRead("https://nseindia.com/content/indices/ind_nifty50list.csv");
StreamReader reader = new StreamReader(stream);
String content = reader.ReadToEnd();
Related
I have the following function written in c#:
public static string FunctionName()
{
try
{
using (var httpClient = new HttpClient())
{
var uri = new Uri("https://www.website.com/api/1/auth/user");
httpClient.BaseAddress = uri;
httpClient.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36");
httpClient.DefaultRequestHeaders.Add("Cookie", "apiKey=dasdasd; auth=authcookie_dasd-c28673189043; id_chat.com=dasdasdad");
return httpClient.GetAsync(uri).Result.ToString();
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message.ToString());
}
return null;
}
where the cookie is added directly to the header. Now I was trying to split this up to add it to a CookieContainer. The source code looks like the following:
public static string FunctionName()
{
try
{
HttpClientHandler handler = new HttpClientHandler();
handler.CookieContainer = new CookieContainer();
Uri target = new Uri("https://www.website.com");
handler.CookieContainer.Add(new Cookie("apiKey", "dasdasd") { Domain = target.Host });
handler.CookieContainer.Add(new Cookie("auth", "authcookie_dasd-c28673189043") { Domain = target.Host });
handler.CookieContainer.Add(new Cookie("id_chat.com", "dasdasdad") { Domain = target.Host });
HttpClient http = new HttpClient(handler);
http.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36");
var response = http.GetAsync("https://www.website.com/api/1/auth/user");
Console.WriteLine(response.Result.ToString());
if (response.Result.StatusCode == HttpStatusCode.OK)
{
var json = JsonConvert.DeserializeObject<LoginResponse>(response.Result.Content.ReadAsStringAsync().Result);
return json.id;
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message.ToString());
}
return null;
}
The first function would return the status "OK".
The second function would return the status "Unauthorized".
What is causing this problem? Did I set up the CookieContainer wrong?
I'm getting exception when i run this code
Exception "header must be modified using the appropriate property or method."
HtmlAgilityPack.HtmlWeb web = new HtmlWeb();
web.UserAgent="Mozilla/5.0 (Windows NT 10.0; Win64; x64)
AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36";
web.PreRequest += (request) =>
{
request.Headers.Add("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
request.Headers.Add("Accept-Language", "de-DE");
return true;
};
HtmlAgilityPack.HtmlDocument doc = new
HtmlAgilityPack.HtmlDocument();
doc = web.Load("http://www.alfatah.pk/");
This works for me in https://dotnetfiddle.net/AQbs3v :
HttpWebRequest request = (HttpWebRequest)WebRequest.Create("http://www.alfatah.pk/");
request.UserAgent = "Mozilla / 5.0(Windows NT 10.0; Win64; x64) AppleWebKit / 537.36(KHTML, like Gecko) Chrome / 70.0.3538.77 Safari / 537.36";
request.Accept= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
//request.Headers.Add(HttpRequestHeader.AcceptLanguage, "de-DE");
using (var response = (HttpWebResponse)(request.GetResponse()))
{
HttpStatusCode code = response.StatusCode;
if (code == HttpStatusCode.OK)
{
using (StreamReader sr = new StreamReader(response.GetResponseStream(), Encoding.UTF8))
{
HtmlDocument htmlDoc = new HtmlDocument();
htmlDoc.OptionFixNestedTags = true;
htmlDoc.Load(sr);
Console.Write(htmlDoc.DocumentNode.InnerText);
}
}
}
private void download_nse()
{
string source_location = "https://www.nseindia.com/content/historical/DERIVATIVES/2015/DEC/fo23DEC2015bhav.csv.zip";
Uri uu = new Uri(source_location);
using (WebClient fileReader = new WebClient())
{
try
{
var ua = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36";
fileReader.Headers.Add(HttpRequestHeader.UserAgent, ua);
fileReader.Headers["Accept"] = "/";
fileReader.DownloadFile(uu, #"d:\nse\notworking.zip");
fileReader.Dispose();
}
catch
{
}
}
source_location = "https://www.nseindia.com/content/historical/DERIVATIVES/2016/JAN/fo05JAN2016bhav.csv.zip";
using (WebClient fileReader = new WebClient())
{
try
{
var ua = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36";
fileReader.Headers.Add(HttpRequestHeader.UserAgent, ua);
fileReader.Headers["Accept"] = "/";
fileReader.DownloadFile(source_location, #"d:\nse\working.zip");
}
catch
{
}
}
}
I've checked the url you gave and see that when you select the "Date(DD-MM-YYYY)" and "Select Report" drop-down options and click on "Get Data", there is a GET request send to https://www.nseindia.com/ArchieveSearch with three parameter like this: "?h_filetype=fobhav&date=02-04-2018§ion=FO". And this GET request returns:
<p class="archive_title">F&O - Bhavcopy for 02-04-2018 </p><br>
<br><table cellpadding=5>
<tr>
<td class=t0><a href=/content/historical/DERIVATIVES/2018/APR/fo02APR2018bhav.csv.zip target=new>fo02APR2018bhav.csv.zip</a></td></tr>
<tr>
</table>
</td></tr>
</table>
which contains the link to download file. so when you send the POST request without the GET it fails.
You can send a GET request first and then send POST with the returned link like this:
string source_location2 = "https://www.nseindia.com/ArchieveSearch";
Uri uu2 = new Uri(source_location2);
using (WebClient fileReader = new WebClient())
{
try
{
var ua = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36";
fileReader.Headers.Add(HttpRequestHeader.UserAgent, ua);
fileReader.Headers["Accept"] = "/";
fileReader.QueryString.Add("h_filetype", "fobhav");
fileReader.QueryString.Add("date", "02-04-2018");
fileReader.QueryString.Add("section", "FO");
var response = fileReader.DownloadString(uu2);
//using Html Agility Pack to parse the response and get the download link. you need to add this package through nuget package manager to project for this code to work.
var htmlDoc = new HtmlDocument();
htmlDoc.LoadHtml(response);
var fileLink = htmlDoc.DocumentNode.SelectSingleNode("//table//tr//td//a").Attributes["href"].Value;
//now sending the actual file request...
var fileReader2 = new WebClient();
fileReader2.Headers.Add(HttpRequestHeader.UserAgent, ua);
fileReader2.Headers["Accept"] = "/";
fileReader2.DownloadFile(new Uri("https://www.nseindia.com" + fileLink), #"d:\notworking.zip");
fileReader2.Dispose();
}
catch(Exception e)
{
throw e;
}
}
I have a problem using the response of HttpRequest() i get the response but just the html not the headers and the key that i am searching is on the header so this is my code
HttpRequest rq = new HttpRequest();
rq.Cookies = new CookieDictionary();
rq.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36";
rq.AllowAutoRedirect = true;
rq.IgnoreProtocolErrors = true;
rq.ConnectTimeout = TimeOut;
rq.KeepAlive = true;
var str = rq.Get("url").ToString();
if(str.Contains("404")){
}
i hope you can help me
I Found the answer thanks for your help
var req = rq.Get("url");
if(req.StatusCode.ToString().Contains("NotFound") ){
}
I've used the following code to perform asynchronous HTTP request in C#.
private static Task GetUrl(string url)
{
var request = (HttpWebRequest)WebRequest.Create(url);
request.UserAgent =
"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.69 Safari/537.36";
request.Accept = "text/html";
return Task
.Factory
.FromAsync<WebResponse>(request.BeginGetResponse, request.EndGetResponse, url)
.ContinueWith(t =>
{
if (t.IsCompleted)
{
using (var stream = t.Result.GetResponseStream())
{
using (var reader = new StreamReader(stream))
{
//Console.WriteLine("-- Successfully downloaded {0} --", t.AsyncState);
//Console.WriteLine(reader.ReadToEnd());
}
}
}
else if (t.IsFaulted)
{
Console.WriteLine("There was an error downloading {0} - {1}", t.AsyncState, t.Exception);
}
});
}
However I'm not sure how I should modify the above code to support HTTP post as well. Any help is appreciated!
In particular I'd like to know how I should add BeginGetRequestStream and EndGetRequestStream into the current function...
Use the Method property of the HttpWebRequest:
request.Method = "POST";