Downloading file from redirecting URLs - c#

I am trying to download mp3 from http://www.audiodump.com/. The site has a lot of redirections. However I managed getting a part of it working.
This is my method for getting all informations such as DL links, titles, mp3 durations.
private void _InetGetHTMLSearch(string sArtist)
{
if(_AudioDumpQuery == string.Empty)
{
//return string.Empty;
}
string[] sStringArray;
string sResearchURL = "http://www.audiodump.biz/music.html?" + _AudioDumpQuery + sArtist.Replace(" ", "+");
string aRet;
HttpWebRequest webReq = (HttpWebRequest)HttpWebRequest.Create(sResearchURL);
webReq.Referer = "http://www.audiodump.com/";
try
{
webReq.CookieContainer = new CookieContainer();
webReq.Method = "GET";
using (WebResponse response = webReq.GetResponse())
{
using (Stream stream = response.GetResponseStream())
{
StreamReader reader = new StreamReader(stream);
aRet = reader.ReadToEnd();
//Console.WriteLine(aRet);
string[] aTable = _StringBetween(aRet, "<BR><table", "table><BR>", RegexOptions.Singleline);
if (aTable != null)
{
string[] aInfos = _StringBetween(aTable[0], ". <a href=\"", "<a href=\"");
if (aInfos != null)
{
for(int i = 0; i < aInfos.Length; i++)
{
aInfos[i] = aInfos[i].Replace("\">", "*");
aInfos[i] = aInfos[i].Replace("</a> (", "*");
aInfos[i] = aInfos[i].Remove(aInfos[i].Length - 2);
sStringArray = aInfos[i].Split('*');
aLinks.Add(sStringArray[0]);
aTitles.Add(sStringArray[1]);
sStringArray[2] = sStringArray[2].Replace("`", "'");
sStringArray[2] = sStringArray[2].Replace("dont", "don't");
sStringArray[2] = sStringArray[2].Replace("lets", "let's");
sStringArray[2] = sStringArray[2].Replace("cant", "can't");
sStringArray[2] = sStringArray[2].Replace("shes", "she's");
sStringArray[2] = sStringArray[2].Replace("aint", "ain't");
sStringArray[2] = sStringArray[2].Replace("didnt", "didn't");
sStringArray[2] = sStringArray[2].Replace("im", "i'm");
sStringArray[2] = sStringArray[2].Replace("youre", "you're");
sStringArray[2] = sStringArray[2].Replace("ive", "i've");
sStringArray[2] = sStringArray[2].Replace("youll", "you'll");
sStringArray[2] = sStringArray[2].Replace("'", "'");
sStringArray[2] = sStringArray[2].Replace("'", "simplequotes");
sStringArray[2] = sStringArray[2].Replace("vk.com", "");
sStringArray[2] = _StringReplaceCyrillicChars(sStringArray[2]);
sStringArray[2] = Regex.Replace(sStringArray[2], #"<[^>]+>| ", "").Trim();
sStringArray[2] = Regex.Replace(sStringArray[2], #"\s{2,}", " ");
sStringArray[2] = sStringArray[2].TrimStart('\'');
sStringArray[2] = sStringArray[2].TrimStart('-');
sStringArray[2] = sStringArray[2].TrimEnd('-');
sStringArray[2] = sStringArray[2].Replace("- -", "-");
sStringArray[2] = sStringArray[2].Replace("http", "");
sStringArray[2] = sStringArray[2].Replace("www", "");
sStringArray[2] = sStringArray[2].Replace("mp3", "");
sStringArray[2] = sStringArray[2].Replace("simplequotes", "'");
aDurations.Add(sStringArray[2]);
}
}
else
{
//Console.WriteLine("Debug");
}
}
else
{
//Console.WriteLine("Debug 2");
}
//return aRet;
}
}
}
catch (Exception ex)
{
//return null;
////Console.WriteLine("Debug message: " + ex.Message);
}
}
I simply had to add referrer to prevent the search from redirection webReq.Referer = "http://www.audiodump.com/";
However when I want to download the mp3 I can't get it working. The urls are correct and checked with the ones I get when I download them manually rather than programmatically.
This is my mp3 download part:
private void _DoDownload(string dArtist, ref string dPath)
{
if (!Contain && skip <= 3 && !Downloading)
{
Random rnd = new Random();
int Link = rnd.Next(5);
_InetGetHTMLSearch(dArtist);
Console.WriteLine("--------------------------------> " + aLinks[0]);
string path = mp3Path + "\\" + dArtist + ".mp3";
if (DownloadOne(aLinks[Link], path, false))
{
hTimmer.Start();
Downloading = true;
}
}
else if (Downloading)
{
int actualBytes = strm.Read(barr, 0, arrSize);
fs.Write(barr, 0, actualBytes);
bytesCounter += actualBytes;
double percent = 0d;
if (fileLength > 0)
percent =
100.0d * bytesCounter /
(preloadedLength + fileLength);
label1.Text = Math.Round(percent).ToString() + "%";
if (Math.Round(percent) >= 100)
{
string path = mp3Path + "\\" + dArtist + ".mp3";
label1.Text = "";
dPath = path;
aLinks.Clear();
hTimmer.Stop();
hTimmer.Reset();
fs.Flush();
fs.Close();
lastArtistName = "N/A";
Downloading = false;
}
if (Math.Round(percent) <= 1)
{
if (hTimmer.ElapsedMilliseconds >= 3000)
{
string path = mp3Path + "\\" + dArtist + ".mp3";
hTimmer.Stop();
hTimmer.Reset();
fs.Flush();
fs.Close();
File.Delete(path);
Contain = false;
skip += 1;
Downloading = false;
}
}
}
}
private static string ConvertUrlToFileName(string url)
{
string[] terms = url.Split(
new string[] { ":", "//" },
StringSplitOptions.RemoveEmptyEntries);
string fname = terms[terms.Length - 1];
fname = fname.Replace('/', '.');
return fname;
} //ConvertUrlToFileName
private static long GetExistingFileLength(string filename)
{
if (!File.Exists(filename)) return 0;
FileInfo info = new FileInfo(filename);
return info.Length;
} //GetExistingFileLength
private static bool DownloadOne(string url, string existingFilename, bool quiet)
{
ServicePointManager.DefaultConnectionLimit = 20;
HttpWebRequest webRequest;
HttpWebResponse webResponse;
IWebProxy proxy = null; //SA???
//fmt = CreateFormat(
//"{0}: {1:#} of {2:#} ({3:g3}%)", "#");
try
{
fname = existingFilename;
if (fname == null)
fname = ConvertUrlToFileName(url);
if (File.Exists(existingFilename))
{
File.Delete(existingFilename);
}
webRequest = (HttpWebRequest)WebRequest.Create(url);
webRequest.UserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A";
webRequest.Referer = "http://www.audiodump.com/";
preloadedLength = GetExistingFileLength(fname);
if (preloadedLength > 0)
webRequest.AddRange((int)preloadedLength);
webRequest.Proxy = proxy; //SA??? or DefineProxy
webResponse = (HttpWebResponse)webRequest.GetResponse();
fs = new FileStream(fname, FileMode.Append, FileAccess.Write);
fileLength = webResponse.ContentLength;
strm = webResponse.GetResponseStream();
if (strm != null)
{
bytesCounter = preloadedLength;
return true;
}
else
{
return false;
}
}
catch (Exception e)
{
//Console.WriteLine(
//"{0}: {1} '{2}'",
// url, e.GetType().FullName,
//e.Message);
return false;
}
//exception
} //DownloadOne
The method _DoDownload() is executed from a timer which runs every 250 milliseconds. This way works perfectly on other sites. However audiodump is giving me hard time with these redirections.
I am not a genius with httprequest. I managed solving the search issue however the download part is freaking me out. Any advice on how to manage the download issue?

You just need to set referrer to the page from where you got that download link. For example you grabbed links to files from page "http://www.audiodump.biz/music.html?q=whatever", then when downloading file set that as Referrer, not just "http://www.audiodump.biz".

Related

How to fix a problem with inserting data twice

I have a code that upload files by an fileuploader, as well as a InsertCommand which inserts a data, belonging to that file.
That works fine, but if the string "maxId" has no result, he runs the button-code twice, upload the file twice and finally insert the data twice.
Any Ideas?
See my following code:
if (Bericht_Hochladen.HasFile)
{
SqlCommand commandMaxActionDocID = new SqlCommand("SELECT MAX(actiondoc_id) FROM tbl_action WHERE SUBSTRING(actiondoc_nr, 2, 7) ='" + newNumber.Substring(newNumber.Length - 7) + "'", conn);
string maxId = commandMaxActionDocID.ExecuteScalar().ToString();
if (maxId == "")
{
neue_nr = newNumber.ToString() + "_" + DateTime.Now.ToString("yyyy-MM-dd", ci) + "_" + docart + "_DOC001";
}
else
{
SqlCommand commandMaxActionDocNR = new SqlCommand("SELECT actiondoc_nr FROM tbl_action_1100 WHERE actiondoc_id = #maxId", conn);
commandMaxActionDocNR.Parameters.AddWithValue("#maxId", maxId.ToString());
string NrMitT = commandMaxActionDocNR.ExecuteScalar().ToString();
string count_str = NrMitT.Substring(NrMitT.Length - 3, 3);
int count_int = Int32.Parse(count_str);
count_int = count_int + 1;
count_str = count_int.ToString("000");
neue_nr = newNumber.ToString() + "_" + DateTime.Now.ToString("yyyy-MM-dd", ci) + "_" + docart + "_DOC" + count_str;
}
string dateiendung = Path.GetExtension(Bericht_Hochladen.FileName);
string todaydate = DateTime.Today.ToString("yyyy-MM-dd");
string dokumentname = neue_nr;
#region [Datenupload WebDAV]
Bericht_Hochladen.PostedFile.SaveAs(HttpContext.Current.Server.MapPath("~/TemporaryUploadedFiles/") + dokumentname + dateiendung);
SqlCommand commandVerweis = new SqlCommand("SELECT pfad_adresse FROM tbl_sys_pfad WHERE pfad_nr = 102", conn);
string verweis = (string)commandVerweis.ExecuteScalar();
//temporäreres Speichern auf dem WebServer
FileStream fstream = new FileStream(HttpContext.Current.Server.MapPath("~/TemporaryUploadedFiles/") + dokumentname + dateiendung, FileMode.OpenOrCreate, FileAccess.Read);
HttpWebRequest request_folder = (HttpWebRequest)WebRequest.Create(filepath);
request_folder.Credentials = new NetworkCredential(szUsername, szPassword);
try
{
// Specify the MKCOL method.
request_folder.Method = "MKCOL";
HttpWebResponse response_folder = (HttpWebResponse)request_folder.GetResponse();
Response.Close();
}
catch (Exception ex)
{
//MessageBox.Show(ex.ToString());
}
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(filepath + dokumentname + dateiendung);
request.Credentials = new NetworkCredential(szUsername, szPassword);
try
{
request.Method = #"PUT";
request.ContentLength = fstream.Length;
request.AllowWriteStreamBuffering = true;
request.SendChunked = false;
request.KeepAlive = false;
Stream request_stream = request.GetRequestStream();
byte[] indata = new byte[1024];
int bytes_read = fstream.Read(indata, 0, indata.Length);
while (bytes_read > 0)
{
request_stream.Write(indata, 0, indata.Length);
bytes_read = fstream.Read(indata, 0, indata.Length);
}
fstream.Close();
request_stream.Close();
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
if (response.StatusCode != HttpStatusCode.OK && response.StatusCode != HttpStatusCode.Created)
{
//MessageBox.Show("Couldn't upload file");
}
}
catch (Exception ex)
{
//MessageBox.Show(ex.ToString());
}
string del_URL = HttpContext.Current.Server.MapPath("~/TemporaryUploadedFiles/") + dokumentname + dateiendung;
File.Delete(del_URL);
#endregion
SqlCommand commandDocUpload = new SqlCommand("INSERT INTO tbl_action(actiondoc_action_id, actiondoc_nr, actiondoc_art, actiondoc_pfad, actiondoc_datum, actiondoc_endung, actiondoc_bem, sys_crt_user, sys_crt_timestamp, sys_deleted) VALUES (#actiondoc_action_id, #actiondoc_nr, #actiondoc_art, #actiondoc_pfad, #actiondoc_datum, #actiondoc_endung, #actiondoc_bem, #sys_crt_user, #sys_crt_timestamp, #sys_deleted)", conn);
commandDocUpload.Parameters.Add(new SqlParameter("#actiondoc_action_id", (string)Session["action_id"].ToString()));
commandDocUpload.Parameters.Add(new SqlParameter("#actiondoc_nr", neue_nr.ToString()));
commandDocUpload.Parameters.Add(new SqlParameter("#actiondoc_art", Session["dokument_art_volltext"].ToString()));
commandDocUpload.Parameters.Add(new SqlParameter("#actiondoc_pfad", neue_nr.ToString() + dateiendung.ToString()));
commandDocUpload.Parameters.Add(new SqlParameter("#actiondoc_datum", DateTime.Now.ToString("yyyy-MM-dd", ci)));
commandDocUpload.Parameters.Add(new SqlParameter("#actiondoc_endung", dateiendung.ToString()));
commandDocUpload.Parameters.Add(new SqlParameter("#actiondoc_bem", txtBeschreibungDoc.Text));
commandDocUpload.Parameters.Add(new SqlParameter("#sys_crt_user", (string)Session["login"].ToString()));
commandDocUpload.Parameters.Add(new SqlParameter("#sys_crt_timestamp", DateTime.Now.ToString("dd.MM.yyyy HH:mm:ss", ci)));
commandDocUpload.Parameters.Add(new SqlParameter("#sys_deleted", "0"));
commandDocUpload.ExecuteNonQuery();
conn.Close();
if (IsPostBack)
{
//Do Something
return;
}
}
Try changing this line:
string maxId = commandMaxActionDocID.ExecuteScalar().ToString();
To this:
string maxId = Convert.ToString(commandMaxActionDocID.ExecuteScalar());
And for reference:
Difference between Convert.ToString() and .ToString()
I think what's going on is that your ExecuteScalar() is returning something the if statement doesn't recognize, and therefore it's just skipping over it, and then once the data gets added, the value gets populated. Do you have the code that calls the method? That might be helpful.
EDIT:
David, Try changing your control to this:
<div class="pull-left btn-mybtn"> <asp:Button ID="btnDateiHochladen" runat="server" Text="Hochladen" CssClass="font-regular btn-mybtn pull-right"/> </div>
Also, does your code behind (C#) have an OnCommand or OnClick event for this particular button?

How to add a contact to Yahoo using Oauth-2 in Asp.Net C#

I've been wrestling with this for a while now and can't seem to find a solution. The closest I've come is PHP code for Oauth-1 by Joe Chung (https://github.com/joechung/oauth_yahoo), but I can't get my head wrapped around it.
I'm using Asp.Net, and this code is in the ContactController. I have no trouble Getting contacts from Yahoo. The problem is Adding a contact to a user's Yahoo address book. The program proceeds through the Yahoo login process, and there are no errors. But the contact is not saved. I hope someone can take a look at this and tell me what I'm missing.
Thanks.
private string AddYahooContact(string responseFromServer, string contactIdForYahoo)
{
// Some of this from http://www.yogihosting.com/implementing-yahoo-contact-reader-in-asp-net-and-csharp/
responseFromServer = responseFromServer.Substring(1, responseFromServer.Length - 2);
string accessToken = "", xoauthYahooGuid = "", refreshToken = "";
string[] splitByComma = responseFromServer.Split(',');
foreach (string value in splitByComma)
{
if (value.Contains("access_token"))
{
string[] accessTokenSplitByColon = value.Split(':');
accessToken = accessTokenSplitByColon[1].Replace('"'.ToString(), "");
}
else if (value.Contains("xoauth_yahoo_guid"))
{
string[] xoauthYahooGuidSplitByColon = value.Split(':');
xoauthYahooGuid = xoauthYahooGuidSplitByColon[1].Replace('"'.ToString(), "");
}
else if (value.Contains("refresh_token"))
{
string[] refreshTokenSplitByColon = value.Split(':');
refreshToken = refreshTokenSplitByColon[1].Replace('"'.ToString(), "");
}
}
// How to build contactUrl from https://developer.yahoo.com/social/rest_api_guide/contacts-resource.html#contacts-xml_request_put
// This is Yahoo's address to add a contact
string contactUrl = "https://social.yahooapis.com/v1/user/" + xoauthYahooGuid + "/contacts";
// Much of this from https://developer.yahoo.com/dotnet/howto-rest_cs.html
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(contactUrl);
webRequest.Method = "POST";
webRequest.Headers["Authorization"] = "Bearer " + accessToken;
webRequest.ContentType = "application/x-www-form-urlencoded"; // Tried "application/x-www-form-urlencoded & application/xml" & "text/xml".
// Create the data we want to send
string yahooContact = BuildYahooContact(contactIdForYahoo);
byte[] byteData = UTF8Encoding.UTF8.GetBytes(yahooContact);
webRequest.ContentLength = byteData.Length;
using (Stream postStream = webRequest.GetRequestStream())
{
postStream.Write(byteData, 0, byteData.Length);
}
responseFromServer = "";
try
{
using (HttpWebResponse response = webRequest.GetResponse() as HttpWebResponse)
{
StreamReader reader = new StreamReader(response.GetResponseStream());
responseFromServer = reader.ReadToEnd();
}
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
return yahooContact;
}
private string BuildYahooContact(string contactIdForYahoo)
{
Guid contactId = Guid.Parse(contactIdForYahoo);
Models.Contact contact = GetContactForId(contactId); // And find the contact
string firstName = "";
string lastName = "";
if (contact.FullName != null)
{
int index = contact.FullName.IndexOf(" ");
if (index > 0)
{
firstName = contact.FullName.Substring(0, index);
lastName = (contact.FullName.Substring(index + 1));
}
else
{
lastName = contact.FullName;
}
}
StringBuilder data = new StringBuilder();
data.Append("<contact>");
data.Append("<fields><type>name</type><value>");
data.Append("<givenName>" + firstName + "</givenName><middleName/>");
data.Append("<familyName>" + lastName + "</familyName>");
data.Append("<prefix/><suffix/><givenNameSound/><familyNameSound/>");
data.Append("</value></fields>");
data.Append("<fields><type>address</type>");
data.Append("<value><street>" + contact.Address + "</street>");
data.Append("<city>" + contact.City + "</city>");
data.Append("<stateOrProvince>" + contact.State + "</stateOrProvince>");
data.Append("<postalCode>" + contact.Zip + "</postalCode>");
data.Append("<country>United States</country>");
data.Append("<countryCode>US</countryCode>");
data.Append("</value></fields>");
data.Append("<fields><type>notes</type><value>" + contact.Note + "</value></fields>");
data.Append("<fields><type>link</type><value>" + contact.Website + "</value></fields>");
data.Append("<fields><type>email</type><value>" + contact.Email + "</value></fields>");
data.Append("<fields><type>phone</type><value>" + contact.BusinessPhone + "</value><flags>WORK</flags></fields>");
data.Append("<fields><type>phone</type><value>" + contact.BestPhone + "</value><flags>MOBILE</flags></fields>");
data.Append("<fields><type>phone</type><value>" + contact.SecondPhone + "</value><flags>PERSONAL</flags></fields>");
data.Append("<categories><category><name>GoGoContract</name></category></categories>");
data.Append("</contact>");
return data.ToString();
}

System.Net.WebException while downloading file

I have a list of mp3 which I am downloading. After some files are downloaded, not all of them - around 5-7, I get WebException. I did a stacktrace and this is the result.
Exception thrown: 'System.Net.WebException' in System.dll
Debug message: The operation has timed out
InnerEx: at System.Net.HttpWebRequest.GetResponse()
at iBlock.Main._InetGetHTMLSearch(String sArtist) in C:\Users\...\Main.cs:line 590
My _InetGetHTMLSearch looks like this
private void _InetGetHTMLSearch(string sArtist)
{
aLinks.Clear();
if (AudioDumpQuery == string.Empty)
{
//return string.Empty;
}
string[] sStringArray;
string sResearchURL = "http://www.audiodump.biz/music.html?" + AudioDumpQuery + sArtist.Replace(" ", "+");
string aRet;
HttpWebRequest webReq = (HttpWebRequest)HttpWebRequest.Create(sResearchURL);
webReq.UserAgent = "Mozilla / 5.0(Macintosh; Intel Mac OS X 10_9_3) AppleWebKit / 537.75.14(KHTML, like Gecko) Version / 7.0.3 Safari / 7046A194A";
webReq.Referer = "http://www.audiodump.com/";
webReq.Timeout = 5000;
try
{
webReq.CookieContainer = new CookieContainer();
webReq.Method = "GET";
using (WebResponse response = webReq.GetResponse())
{
using (Stream stream = response.GetResponseStream())
{
StreamReader reader = new StreamReader(stream);
aRet = reader.ReadToEnd();
//Console.WriteLine(aRet);
string[] aTable = _StringBetween(aRet, "<BR><table", "table><BR>", RegexOptions.Singleline);
if (aTable != null)
{
string[] aInfos = _StringBetween(aTable[0], ". <a href=\"", "<a href=\"");
if (aInfos != null)
{
for (int i = 0; i < aInfos.Length; i++)
{
//do some magic here
}
}
else
{
//debug
}
}
else
{
//debug 2
}
}
response.Dispose();
}
}
catch (Exception ex)
{
Console.WriteLine("Debug message: " + ex.Message + "InnerEx: " + ex.StackTrace);
aLinks.Clear();
return;
//throw exception
}
}
what this method does is simple. A simple search of the sArtist given at audiodump.com
I have a timer which runs very fast, every 10ms.
private void MainTimer_Tick(object sender, EventArgs e)
{
_DoDownload(DoubleDimList[i][y], ref mp3ToPlay);
if (muted) Mute(0);
if (Downloading)
{
StatusLabel.Text = "Downloading: " + DoubleDimList[i][y];
}
}
Now this timer handles the download in the background in a Global scope.
The _DoDownload methos which basically starts the entire process looks like this
private void _DoDownload(string dArtist, ref string dPath)
{
if (!Contain && skip <= 3 && !Downloading)
{
try
{
_InetGetHTMLSearch(dArtist);
if (aLinks.Count < 1)
{
//skip and return
Console.WriteLine("Skipping: " + dArtist);
IniWriteValue(_playlists[i], "Track " + y, dArtist + " -iBlockSkip");
y++;
return;
}
string path = mp3Path + "\\" + dArtist + ".mp3";
if (DownloadOne(aLinks[0], path, false))
{
hTimmer.Start();
Downloading = true;
}
}
catch (Exception Ex)
{
MessageBox.Show("Download start error: " + Ex.Message);
}
}
else if (Downloading)
{
try {
int actualBytes = strm.Read(barr, 0, arrSize);
fs.Write(barr, 0, actualBytes);
bytesCounter += actualBytes;
double percent = 0d;
if (fileLength > 0)
percent =
100.0d * bytesCounter /
(preloadedLength + fileLength);
label1.Text = Math.Round(percent) + "%";
if (Math.Round(percent) >= 100)
{
string path = mp3Path + "\\" + dArtist + ".mp3";
label1.Text = "";
dPath = path;
aLinks.Clear();
hTimmer.Stop();
hTimmer.Reset();
fs.Flush();
fs.Close();
lastArtistName = "N/A";
Downloading = false;
y++;
if (y >= DoubleDimList[i].Count)
{
i++;
}
}
if (Math.Round(percent) <= 1)
{
if (hTimmer.ElapsedMilliseconds >= 3000)
{
string path = mp3Path + "\\" + dArtist + ".mp3";
hTimmer.Stop();
hTimmer.Reset();
fs.Flush();
fs.Close();
System.IO.File.Delete(path);
Contain = false;
skip += 1;
Downloading = false;
}
} }
catch(Exception Ex)
{
MessageBox.Show("Downloading error: " + Ex.Message);
}
}
}
Now once the exception is thrown it messes up the entire project. As you see in the last method, if _InetGetHTMLSearch doesn't update the search(returns nothing) I am skipping and moving to next search. However the exception will be thrown in every next search. I tried setting new cookies in every search but still didn't work.
Any solutions how to avoid this issue?
P.S. I have to say that if I change the timer's Interval to 500ms it will download more mp3 before the exception is thrown but not all of them.
Edit: The issue here is obvious. The request timesout but even if I set it to Timeout.Infinite it will hand there forever

Convert C# Console application application to Website/WebApp

I've made a console app (don't judge for it being badly written, I can't even do classes properly and I know that RegEx doesn't work with XML/HTML, but let's not mind that I'm not even a programmer)
int retryCount = 0;
int retryCount02 = 0;
int innerpercentage = 1;
HttpWebResponse response = null;
HttpWebResponse response02 = null;
//WebRequest groupingurl = WebRequest.Create("supersecretlink");
WebRequest groupingurl = WebRequest.Create("anothersupersecretlink"); // temp
var groupingurlread = (HttpWebResponse)groupingurl.GetResponse();
using (var groupingreader = new StreamReader(groupingurlread.GetResponseStream(), Encoding.ASCII))
{
string responseGrouping = groupingreader.ReadToEnd();
Console.WriteLine("--------");
var pattern = #"""ClassID"" Value=""(.*?)""";
var matches = Regex.Matches(responseGrouping, pattern)
.OfType<Match>()
.Select(m => m.Groups[1].Value)
.ToArray();
int percentage = 1;
int percentagedone = matches.Length;
foreach (string r in matches)
{
while (true)
{
try
{
var url = "themostseriouslinkever";
string req = File.ReadAllText(#"somanylinksrighthere", Encoding.Default);
var go = url + req;
go = String.Format(go, r);
WebRequest request = WebRequest.Create(go);
response = (HttpWebResponse)request.GetResponse();
break;
}
catch (WebException)
{
Console.WriteLine("Retry No. {0}", retryCount+1);
if (++retryCount < 3) continue;
throw;
}
}
retryCount = 0;
using (var reader = new StreamReader(response.GetResponseStream(), Encoding.ASCII))
{
string responseText = reader.ReadToEnd();
pattern = #"<PartNumber>(.*?)</PartNumber>";
matches = Regex.Matches(responseText, pattern)
.OfType<Match>()
.Select(m => m.Groups[1].Value)
.ToArray();
int innerpercentagedone = matches.Length;
foreach (string bbb in matches)
{
// -----
//Console.WriteLine(bbb);
while (true)
{
try
{
var url02 = "anothersecret";
string req02 = File.ReadAllText(#"seeecreet", Encoding.Default);
var go02 = url02 + req02;
go02 = string.Format(go02, bbb);
WebRequest request02 = WebRequest.Create(go02);
response02 = (HttpWebResponse)request02.GetResponse();
break;
}
catch (WebException)
{
Console.WriteLine("Retry No. {0}", retryCount02+1);
if (++retryCount02 < 3) continue;
throw;
}
}
retryCount02 = 0;
using (var reader02 = new StreamReader(response02.GetResponseStream(), Encoding.ASCII))
{
string responseText02 = reader02.ReadToEnd();
responseText02 = Regex.Replace(responseText02, (#"<\?xml.*\?>"), "");
//Console.WriteLine(responseText02);
//Console.WriteLine("</PartNumber>.*<");
responseText = Regex.Replace(responseText, ((#"<PartNumber>" + bbb + #"<\/PartNumber>")), responseText02);
//Console.WriteLine(responseText);
Console.Clear();
Console.WriteLine("(" + percentage + ") " + innerpercentage + "/" + innerpercentagedone + ". Total: " + percentage + " / " + percentagedone);
innerpercentage++;
}
}
//string mydocpath = Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments);
using (StreamWriter outputFile = new StreamWriter(#"D:\Downloads\edasltxmlrequest\ConsoleApplication1\ConsoleApplication1\bin\Debug\test\" + r + ".xml"))
{
outputFile.WriteLine(responseText);
}
//Console.Clear();
//Console.WriteLine(percentage + " / " + percentagedone);
percentage++;
innerpercentage = 1;
//Console.WriteLine(responseText);
}
}
Console.WriteLine("--------");
Console.ReadLine();
}
}
}
So I wanted to ask if it's possible to make it run as a web app (http://www.amazingwebsite.com/ThisThingIWroteRightThere.php

ASPX get response headers back

I would appreciate some help because I am really new to C#. I need to return the response headers back, I need the 3 numbers of status code only actually. The code of the page for the web service is `
<%# Page Language="C#" ContentType="application/json;charset=utf-8"%>
<%# Import Namespace="System.Net" %>
<%# Import Namespace="System.IO" %>
<%# Import Namespace="System.Xml" %>
<%# Import Namespace="System.Web" %>
<%# Import Namespace="Newtonsoft.Json" %>
<%# Import Namespace="Newtonsoft.Json.Linq" %>
<%# Import Namespace="System.Collections.Generic" %>
<%# Import Namespace="log4net" %>
<script runat="server">
static readonly ILog m_Log = LogManager.GetLogger("getWebRequest");
String appUrlEncoded = "application/x-www-form-urlencoded";
String appJson = "application/json";
String textXml = "text/xml";
String appXml = "application/xml";
String textPlain = "text/plain";
/************************ fetchURL *******************/
String fetchURL(string url, string protocol, string enctype, string parameters,
string readWriteTimeout, string conTimeout,
string userName, string password, JObject CustomHeaders, JToken JsonContent)
{
m_Log.Debug("fetchURL(" + url + ", " + protocol + ", " + enctype + ", " + parameters + ", " +
readWriteTimeout + ", " + conTimeout + ", " + userName + ", " + password + ") in");
String result = "";
String method ="GET";
String loginCredentials = userName + ":" + password;
// Accepting self-signed certificates
ServicePointManager.ServerCertificateValidationCallback += delegate(
object
sender,
System.Security.Cryptography.X509Certificates.X509Certificate
pCertificate,
System.Security.Cryptography.X509Certificates.X509Chain pChain,
System.Net.Security.SslPolicyErrors pSSLPolicyErrors)
{
return true;
};
try
{
if (protocol.EndsWith("get") || protocol.EndsWith("delete"))
{
url = url + (String.IsNullOrEmpty(parameters) ? "" : ("?" + parameters));
}
method = protocol.ToUpper();
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
request.Method = method;
request.ContentType = enctype;
request.Timeout = Int32.Parse(conTimeout);
request.ReadWriteTimeout = Int32.Parse(readWriteTimeout);
if (loginCredentials.Length>1)
{
request.Headers.Add("Authorization", "Basic " + Convert.ToBase64String(new ASCIIEncoding().GetBytes(loginCredentials)));
}
if (CustomHeaders != null)
{
//Custom HTTP headers
JEnumerable<JProperty> children = CustomHeaders.Children<JProperty>();
foreach (JProperty child in children)
{
string strKey = child.Name;
string strValue = "";
if (child == null || child.Value == null)
continue;
if (child.Value.Type.Equals(JsonTokenType.String))
{
strValue = (String)((JValue)child.Value).Value;
}
else
{
strValue = child.Value.ToString();
}
m_Log.Debug("request key: '" + strKey + "' -- value: '" + strValue + "'");
if (strKey.Trim().Equals("Accept"))
{
request.Accept = strValue;
}
else if (strKey.Trim().Equals("Connection"))
{
request.Connection = strValue;
}
else if (strKey.Trim().Equals("Content-Length"))
{
request.ContentLength = strValue.Length;
}
else if (strKey.Trim().Equals("Content-Type"))
{
request.ContentType = strValue;
}
else if (strKey.Trim().Equals("Expect"))
{
request.Expect = strValue;
}
else if (strKey.Trim().Equals("If-Modified-Since"))
{
request.IfModifiedSince = DateTime.Parse(strValue);
}
else if (strKey.Trim().Equals("Referer"))
{
request.Referer = strValue;
}
else if (strKey.Trim().Equals("Transfer-Encoding"))
{
request.TransferEncoding = strValue;
}
else if (strKey.Trim().Equals("User-Agent"))
{
request.UserAgent = strValue;
}
else
{
request.Headers.Add(strKey, strValue);
}
}
}
if (method.Equals("POST") || method.Equals("PUT"))
{
using (Stream writeStream = request.GetRequestStream())
{
UTF8Encoding encoding = new UTF8Encoding();
if (enctype.Equals(appJson))
{
if (JsonContent != null)
{
if (JsonContent is JObject)
{
JObject obj = (JObject)JsonContent;
m_Log.Debug(enctype + " encoding JObject: " + obj.ToString());
byte[] bytes = encoding.GetBytes(obj.ToString());
writeStream.Write(bytes, 0, bytes.Length);
}
else
{
JObject obj = new JObject();
obj.Add(new JProperty("content", JsonContent));
m_Log.Debug(enctype + " encoding: " + obj.ToString());
byte[] bytes = encoding.GetBytes(obj.ToString());
writeStream.Write(bytes, 0, bytes.Length);
}
}
}
else if (enctype.Equals(appUrlEncoded))
{
m_Log.Debug(enctype + " encoding parameters: " + parameters);
byte[] bytes = encoding.GetBytes(parameters);
writeStream.Write(bytes, 0, bytes.Length);
}
writeStream.Close();
}
}
HttpWebResponse webResponse = (HttpWebResponse)request.GetResponse();
StreamReader input = new StreamReader(webResponse.GetResponseStream());
result = "";
result += convertToJson(input, webResponse.ContentType);
}
catch (Exception e)
{
Dictionary<string, string> d1 = new Dictionary<string, string>();
string value = "error.com.genesyslab.composer.servererror message= " + e.Message.ToString();
d1.Add("errorMsg", value);
result = Newtonsoft.Json.JavaScriptConvert.SerializeObject(d1);
Response.AppendToLog("GeneralException:" + result);
}
m_Log.Debug("result: " + result);
m_Log.Debug("fetchURL() out");
return result;
}
string parseResultData(StreamReader reader, JsonTextReader jsonReader, string initialData)
{
m_Log.Debug("parseResultData(" + initialData + ") In");
string json = "";
try
{
jsonReader.Read();
// JSON string
json += initialData;
Response.AppendToLog("ContentTypeUnkJSON");
m_Log.Debug("parseResultData() Out");
return json;
}
catch (JsonReaderException)
{
m_Log.Debug("parseResultData() JsonReaderException");
// not a valid JSON - check for XML
try
{
XmlDocument doc = new XmlDocument();
doc.LoadXml(initialData);
reader.Close();
json = Newtonsoft.Json.JavaScriptConvert.SerializeXmlNode(doc.DocumentElement);
Response.AppendToLog("ContentTypeUnkXML");
m_Log.Debug("parseResultData() Out");
return json;
}
catch (Exception e)
{
m_Log.Debug("parseResultData() Exception: " + e.Message);
Response.AppendToLog("ContentTypeUnkTEXT");
Response.AppendToLog("Exception Occured: " +e.Message.ToString());
Dictionary<string, string> d1 = new Dictionary<string, string>();
d1.Add("result", initialData);
string jsonText = Newtonsoft.Json.JavaScriptConvert.SerializeObject(d1);
return jsonText;
}
}
}
/**************convertToJson****************************/
string convertToJson(StreamReader reader, string contentType)
{
m_Log.Debug("convertToJson(" + contentType + ") In");
string json = "";
string data = reader.ReadToEnd();
// Parse into a JSON string
TextReader txReader = new StringReader(data);
Newtonsoft.Json.JsonTextReader jsonReader = new JsonTextReader(txReader);
if (contentType != null && contentType.Length != 0)
{
Response.AppendToLog("Content-Type:" + contentType.ToString());
if (contentType.ToLower().StartsWith(textXml) ||
contentType.ToLower().StartsWith(appXml))
{
try
{
XmlDocument doc = new XmlDocument();
doc.LoadXml(data);
reader.Close();
json = Newtonsoft.Json.JavaScriptConvert.SerializeXmlNode(doc.DocumentElement);
m_Log.Debug("convertToJson() Out");
return json;
}
catch (XmlException e)
{
Response.AppendToLog("ContentTypeXMLFalse");
m_Log.Error("convertToJson() Error in decoding XML: " + e.Message);
throw new XmlException("Error in decoding XML: " + e.Message, e);
}
}
else if (contentType.ToLower().StartsWith(appJson))
{
jsonReader.Read();
// JSON string
json += data;
Response.AppendToLog("ContentTypeJSON");
m_Log.Debug("convertToJson() ContentTypeJSON Out");
return json;
}
else if (contentType.ToLower().StartsWith(textPlain))
{
Response.AppendToLog("ContentTypeTEXT");
Dictionary<string, string> d1 = new Dictionary<string, string>();
d1.Add("result", data);
d1.Add("headers", "");
string jsonText = Newtonsoft.Json.JavaScriptConvert.SerializeObject(d1);
Response.AppendToLog("ContentTypeTEXT:" + jsonText);
m_Log.Debug("convertToJson() ContentTypeTEXT Out");
return jsonText;
}
else
{
Response.AppendToLog("unknown Content-Type:" + contentType);
m_Log.Debug("convertToJson() unknown Content-Type Out");
return (parseResultData(reader, jsonReader, data));
}
}
else
{
Response.AppendToLog("Content-Type NULL");
m_Log.Debug("convertToJson() Content-Type NULL");
return( parseResultData(reader, jsonReader, data) );
}
}
</script>
<%
log4net.Config.XmlConfigurator.Configure();
m_Log.Debug("_________________________________________________");
m_Log.Debug("getWebRequest() In");
// extract parameters
String WebUrl ="";
String Protocol= "";
String EncType = "";
Boolean AuthenAccess = false;
String UserName ="";
String Password ="";
String readWriteTimeout = "20000"; // timeout in milliseconds
String conTimeout = "20000"; // timeout in milliseconds
Stream ins = HttpContext.Current.Request.InputStream;
StreamReader reader = new StreamReader(ins);
string jsonStr = reader.ReadToEnd();
JObject requestObj = JObject.Parse(jsonStr);
m_Log.Debug("requestObj: " + requestObj.ToString());
WebUrl = (string)requestObj["WebUrl"];
Protocol = (string)requestObj["Protocol"];
EncType = (string)requestObj["Enctype"];
AuthenAccess = (Boolean)requestObj["AuthenAccess"];
if (AuthenAccess)
{
UserName = (string)requestObj["UserName"];
Password = (string)requestObj["Password"];
}
// the value passed from the block property overrides the
// global value in the composer.properties
String timeout = (string)requestObj["Timeout"];
if (timeout != null && timeout.Trim().Length > 0)
{
try
{
int timeoutInt = Int32.Parse(timeout);
if (timeoutInt != -1)
{
conTimeout = Convert.ToString(timeoutInt * 1000);
readWriteTimeout = Convert.ToString(timeoutInt * 1000);
}
}
catch (FormatException)
{
// ignore an invalid value
}
}
String ParamStr = "";
int QueryPos = WebUrl.IndexOf('?');
if ((Protocol.EndsWith("get") || Protocol.EndsWith("delete")) && (QueryPos > 0))
{
String QueryString = WebUrl.Substring(QueryPos + 1, WebUrl.Length - (QueryPos + 1));
WebUrl = WebUrl.Substring(0, QueryPos);
String[] Pairs = QueryString.Split('&');
foreach (String Pair in Pairs)
{
string strKey = "";
string strValue = "";
int Pos = Pair.IndexOf('=');
if (Pos == -1)
{
strKey = Pair;
strValue = null;
}
else
{
try
{
strKey = Server.UrlDecode(Pair.Substring(0, Pos));
strValue = Server.UrlDecode(Pair.Substring(Pos + 1, Pair.Length - (Pos + 1)));
}
catch (Exception ex)
{
m_Log.Error("Exception parsing queryString:" + ex.Message);
}
}
if (!ParamStr.Equals(""))
{
ParamStr = ParamStr + "&";
}
ParamStr = ParamStr + Server.UrlEncode(strKey) + "=" + Server.UrlEncode(strValue);
}
}
JObject Parameters = (JObject)requestObj["Parameters"];
if (Parameters != null)
{
JEnumerable<JProperty> children = Parameters.Children<JProperty>();
foreach (JProperty child in children)
{
string strKey = child.Name;
string strValue = "";
if (child == null || child.Value == null)
continue;
if (child.Value.Type.Equals(JsonTokenType.String))
{
strValue = (String)((JValue)child.Value).Value;
}
else
{
strValue = child.Value.ToString();
}
// add to map
if (!ParamStr.Equals(""))
{
ParamStr = ParamStr + "&";
}
ParamStr = ParamStr + Server.UrlEncode(strKey) + "=" + Server.UrlEncode(strValue);
}
}
m_Log.Debug("ParamStr: " + ParamStr);
JObject CustomHeaders = (JObject)requestObj["CustomHeaders"];
JToken JsonContent = (JToken)requestObj["JsonContent"];
//relative path processing
string relativePath = "http://localhost:";
if (WebUrl.StartsWith("."))
{
int slashindex = WebUrl.IndexOf("/");
if (slashindex != -1)
{
int n = WebUrl.Length;
WebUrl = WebUrl.Substring(slashindex + 1, n - slashindex-1);
}
relativePath += HttpContext.Current.Request.ServerVariables["SERVER_PORT"];
relativePath = relativePath + HttpContext.Current.Request.RawUrl.ToString();
int boundary = relativePath.IndexOf("include");
if(boundary !=-1){
relativePath = relativePath.Substring(0, boundary);
}
WebUrl = relativePath + WebUrl;
m_Log.Debug("urlStr: " + WebUrl);
}
m_Log.Debug("WebUrl: " + WebUrl);
if (WebUrl.Length > 0)
{
Response.Write(fetchURL(WebUrl, Protocol, EncType, ParamStr, readWriteTimeout, conTimeout,
UserName, Password, CustomHeaders, JsonContent));
}
m_Log.Debug("getWebRequest() Out");
%>
`
As you can see I have inserted a "header" key-value pair in my returned json repsonse. How do I attach the value of the repsonse headers in the headers property? I have tried webResponse.Headers with no luck.
Thank you very much in advance!

Categories