I have a program that sends/receives POST requests/responses from an online API, similar to a market bot that buys and sells a commodity. It works great, however when i run it, it locks up the current thread and im unable to use anything else in the program. In the future i would also like to make buying and selling asynchronous, so they could happen at the same time. Here is the code that executes, and as you can see it loops continuously until conditions are met:
private void RunBot()
{
numToBuy = (int)nudNumToBuy.Value;
for (int i = 0; i < numToBuy; i++)
{
while (true)
{
if (AttachAndBuy())
{
break;
}
}
}
}
private bool AttachAndBuy()
{
string data = "<DATA HERE>";
string URL = "<URL HERE>";
Cookies.SetCookies(cookie, "PHPSESSID=" + SessionIDTextBox.Text.Replace("PHPSESSID=", ""));
HttpWebRequest Request = (HttpWebRequest)WebRequest.Create(URL);
Request.ContentType = "application/json";
Request.Accept = "*/*";
Request.CookieContainer = Cookies;
Request.Host = "<HOST HERE>";
Request.Method = "POST";
Request.UserAgent = "<USER AGENT HERE>";
Request.Headers.Add("<HEADER>", "<VALUE>");
Request.KeepAlive = true;
byte[] CompressedRequest = SimpleZlib.CompressToBytes(Encoding.UTF8.GetBytes(data), 9);
Stream RequestStream = Request.GetRequestStream();
RequestStream.Write(CompressedRequest, 0, CompressedRequest.Length);
RequestStream.Flush();
Stream CompressedResponseStream = Request.GetResponse().GetResponseStream();
byte[] CompressedResponseData = ReadToEnd(CompressedResponseStream);
string DecompressedResponseData = SimpleZlib.Decompress(CompressedResponseData, null);
OffersResponse Return = Json.Deserialize<OffersResponse>(DecompressedResponseData);
int LowestCost = 1000000000;
Offer BestOffer = new Offer();
foreach (Offer CurrentOffer in Return.data.offers)
{
bool moneyOffer = false;
int Costs = CurrentOffer.requirementsCost;
string id = CurrentOffer._id;
foreach (Requirement CurrentRequirement in CurrentOffer.requirements)
{
if (CurrentRequirement._tpl == "<TEMPLATE ID HERE>")
{
moneyOffer = true;
}
}
if (moneyOffer == false)
{
continue;
}
if (Costs < LowestCost)
{
LowestCost = Costs;
BestOffer = CurrentOffer;
}
}
BestOfferID = BestOffer._id;
BestOfferCost = LowestCost;
string MoneyID = getStack(BestOfferCost);
while (true)
{
BuyRequestAttemptCounter++;
if (LowestCost > 140000)
{
AddLog("No Suitable Item! Skipping! Lowest Item Cost: " + LowestCost.ToString());
return false;
}
else
AddLog("Best Item Cost: " + LowestCost.ToString() + " | ID: " + BestOfferID);
int Result = buyOrder(MoneyID);
if (Result == 0)
{
//log info for averaging
numberPurchased++;
TotalCost += BestOfferCost;
averageCost = TotalCost / numberPurchased;
lblNumPurchased.Text = numberPurchased.ToString();
lblAverageCost.Text = averageCost.ToString();
lstPricesPurchased.Items.Add(LowestCost.ToString());
AddLog("====================================");
AddLog("Number Purchased: " + numberPurchased);
AddLog("Average Cost: " + averageCost);
AddLog("====================================");
System.Media.SystemSounds.Exclamation.Play();
return true;
}
else if (Result == 1)
return false;
else if (Result == 2)
continue;
else
return false;
}
}
private int buyOrder(string MoneyID)
{
string data = "<DATA HERE>";
string URL = "<URL HERE>";
Cookies.SetCookies(cookie, "PHPSESSID=" + SessionIDTextBox.Text.Replace("PHPSESSID=", ""));
HttpWebRequest Request = (HttpWebRequest)WebRequest.Create(URL);
Request.ContentType = "application/json";
Request.Accept = "*/*";
Request.CookieContainer = Cookies;
Request.Host = "<HOST HERE>";
Request.Method = "POST";
Request.UserAgent = "<USER AGENT HERE>";
Request.Headers.Add("<HEADER>", "<VALUE>");
Request.KeepAlive = true;
byte[] CompressedRequest = SimpleZlib.CompressToBytes(Encoding.UTF8.GetBytes(data), 9);
Stream RequestStream = Request.GetRequestStream();
RequestStream.Write(CompressedRequest, 0, CompressedRequest.Length);
RequestStream.Flush();
Stream CompressedResponseStream = Request.GetResponse().GetResponseStream();
byte[] CompressedResponseData = ReadToEnd(CompressedResponseStream);
string DecompressedResponseData = SimpleZlib.Decompress(CompressedResponseData, null);
ResponseRoot Return = Json.Deserialize<ResponseRoot>(DecompressedResponseData);
string returnErrorCode = DecompressedResponseData.ToString();
//AddLog(DecompressedResponseData);
if (Return.err == 0 && returnErrorCode.Contains("id"))
{
System.Windows.Forms.Clipboard.SetText(DecompressedResponseData);
//AddLog("Successful Purchase!");
return 0;
}
else if (returnErrorCode.Contains("1503"))
{
//AddLog("Failed with 1503!");
return 1;
}
else if (returnErrorCode.Contains("1512"))
{
// AddLog("Failed with 1512!");
return 2;
}
return 3;
}
As stated above, ideally i would like to run both the "attachandbuy" and "buyorder" functions at the same time, and then eventually i will add a sell function, also running concurrently. Is this possible? Thanks.
You can fix this by creating a list of tasks and executing them, this stops the main thread from being blocked while a call hasn't returned yet, e.g. -
var tasks = new List<Task>();
for (int i = 0; i < numToBuy; i++)
{
var task = new Task(() =>
{
AttachAndBuy()
});
tasks.Add(task);
task.Start();
}
Task.WaitAll(tasks.ToArray());
(Note: I've not actually tested this code, it's just a rough example)
Misunderstanding below -
You'll want to use some parallel programming for this - https://learn.microsoft.com/en-us/dotnet/standard/parallel-programming/task-based-asynchronous-programming?redirectedfrom=MSDN within your for loop e.g. -
private void RunBot()
{
numToBuy = (int)nudNumToBuy.Value;
for (int i = 0; i < numToBuy; i++)
{
while (true)
{
Parallel.Invoke(() => AttachAndBuy(), () => BuyOrder());
}
}
}
Related
I use code to get data from Atlassian Jira and put the data from the response into a container, how can I make this method asynchronous?
I have tried various methods with webrequest, but they all point to an error in the code, perhaps I do not understand how it works at all. You can suggest documentation or specify in the code what I can do with asynchrony for such a request.
public IEnumerable<JiraDataModel> GetData(string dateFrom, string dateTo)
{
int allTicketsCount = 0;
int devTicketsCount = 0;
int slaTicketsCount = 0;
List<JiraRequestUrl> urlArray = new List<JiraRequestUrl>();
urlArray.AddRange(new List<JiraRequestUrl>
{
new JiraRequestUrl{type = "all", //*data*//},
new JiraRequestUrl(){type = "dev",//*data*//});
try
{
foreach (JiraRequestUrl u in urlArray)
{
WebRequest request = WebRequest.Create(_jiraUrl + u.url);
request.ContentType = "application/json; charset=utf-8";
request.Headers.Add("Authorization", "Basic " + Convert.ToBase64String(Encoding.Default.GetBytes(_credentials)));
request.Headers.Add("maxResults", "100");
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
if (response.StatusCode == HttpStatusCode.OK)
{
var responseString = new StreamReader(response.GetResponseStream()).ReadToEnd();
JiraGetDataModel.RootObject DeserializedResponse = (JiraGetDataModel.RootObject)JsonConvert.DeserializeObject(responseString, typeof(JiraGetDataModel.RootObject));
if (u.type.Equals("all"))
{
allTicketsCount = DeserializedResponse.total;
}
if (u.type.Equals("dev"))
{
devTicketsCount = DeserializedResponse.total;
}
}
}
sorry, I didn't quite understand that I didn't insert my attempts to use async, I did it, and it seems to work, can I improve something in my code? added 2 lines
public async Task<IEnumerable<JiraDataModel>> GetData(string dateFrom, string dateTo)
{
int allTicketsCount = 0;
int devTicketsCount = 0;
int slaTicketsCount = 0;
List<JiraRequestUrl> urlArray = new List<JiraRequestUrl>();
urlArray.AddRange(new List<JiraRequestUrl>
{
new JiraRequestUrl{type = "all", //*data*//},
new JiraRequestUrl(){type = "dev",//*data*//});
try
{
foreach (JiraRequestUrl u in urlArray)
{
WebRequest request = WebRequest.Create(_jiraUrl + u.url);
request.ContentType = "application/json; charset=utf-8";
request.Headers.Add("Authorization", "Basic " + Convert.ToBase64String(Encoding.Default.GetBytes(_credentials)));
request.Headers.Add("maxResults", "100");
HttpWebResponse response = (HttpWebResponse)(await request.GetResponseAsync()request.GetResponse();
if (response.StatusCode == HttpStatusCode.OK)
{
var responseString = new StreamReader(response.GetResponseStream()).ReadToEnd();
JiraGetDataModel.RootObject DeserializedResponse = (JiraGetDataModel.RootObject)JsonConvert.DeserializeObject(responseString, typeof(JiraGetDataModel.RootObject));
if (u.type.Equals("all"))
{
allTicketsCount = DeserializedResponse.total;
}
if (u.type.Equals("dev"))
{
devTicketsCount = DeserializedResponse.total;
}
}
}
I don't understand what may cause this error
I am using the below function with about 1000 concurrent connections
Each connection uses a different webproxy
After a while like 15 minutes working, the established TCP connection count starts to stack and internet connectivity becomes lost
When i do not use any webproxy, i do not encounter any error
I am using below function to retrieve active TCP connections count
var properties = IPGlobalProperties.GetIPGlobalProperties();
I don't see any leak in my function
So i need your help to solve this annoying problem
c# .net 4.6.2
Here the statuses of active TCP Connections when this problem occurs
public static cs_HttpFetchResults func_fetch_Page(
string srUrl, int irTimeOut = 60,
string srRequestUserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:31.0) Gecko/20100101 Firefox/31.0",
string srProxy = null, int irCustomEncoding = 0, bool blAutoDecode = true, bool blKeepAlive = true)
{
cs_HttpFetchResults mycs_HttpFetchResults = new cs_HttpFetchResults();
mycs_HttpFetchResults.srFetchingFinalURL = srUrl;
HttpWebRequest request = null;
WebResponse response = null;
try
{
request = (HttpWebRequest)WebRequest.Create(srUrl);
request.CookieContainer = new System.Net.CookieContainer();
if (srProxy != null)
{
string srProxyHost = srProxy.Split(':')[0];
int irProxyPort = Int32.Parse(srProxy.Split(':')[1]);
WebProxy my_awesomeproxy = new WebProxy(srProxyHost, irProxyPort);
my_awesomeproxy.Credentials = new NetworkCredential();
request.Proxy = my_awesomeproxy;
}
else
{
request.Proxy = null;
}
request.ContinueTimeout = irTimeOut * 1000;
request.ReadWriteTimeout = irTimeOut * 1000;
request.Timeout = irTimeOut * 1000;
request.UserAgent = srRequestUserAgent;
request.KeepAlive = blKeepAlive;
request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
WebHeaderCollection myWebHeaderCollection = request.Headers;
myWebHeaderCollection.Add("Accept-Language", "en-gb,en;q=0.5");
myWebHeaderCollection.Add("Accept-Encoding", "gzip, deflate");
request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;
using (response = request.GetResponse())
{
using (Stream strumien = response.GetResponseStream())
{
Encoding myEncoding = Encoding.UTF8;
string srContentType = "";
if (response.ContentType != null)
{
srContentType = response.ContentType;
if (srContentType.Contains(";"))
{
srContentType = srContentType.Split(';')[1];
}
srContentType = srContentType.Replace("charset=", "");
srContentType = func_Process_Html_Input(srContentType);
}
try
{
myEncoding = Encoding.GetEncoding(srContentType);
}
catch
{
myEncoding = irCustomEncoding == 0 ? Encoding.UTF8 : Encoding.GetEncoding(irCustomEncoding);
}
using (StreamReader sr = new StreamReader(strumien, myEncoding))
{
mycs_HttpFetchResults.srFetchBody = sr.ReadToEnd();
if (blAutoDecode == true)
{
mycs_HttpFetchResults.srFetchBody = HttpUtility.HtmlDecode(mycs_HttpFetchResults.srFetchBody);
}
mycs_HttpFetchResults.srFetchingFinalURL = Return_Absolute_Url(response.ResponseUri.AbsoluteUri.ToString(), response.ResponseUri.AbsoluteUri.ToString());
mycs_HttpFetchResults.blResultSuccess = true;
}
}
}
if (request != null)
request.Abort();
request = null;
}
catch (Exception E)
{
if (E.Message.ToString().Contains("(404)"))
mycs_HttpFetchResults.bl404 = true;
csLogger.logCrawlingErrors("crawling failed url: " + srUrl, E);
}
finally
{
if (request != null)
request.Abort();
request = null;
if (response != null)
response.Close();
response = null;
}
return mycs_HttpFetchResults;
}
My application uses a httpwebrequest to GET certain information from my WebAPI. What I'm trying to do is retry the request if the connection is lost or if there is no connection at all.
public static string httpsGET(string passedweburi, string BCO)
{
string content = "";
//GET method
HttpWebRequest HttpRequest = (HttpWebRequest)WebRequest.Create(passedweburi + BCO);
HttpRequest.Method = "GET";
//Response
HttpWebResponse response = (HttpWebResponse)HttpRequest.GetResponse();
StreamReader sr = new StreamReader(response.GetResponseStream(), System.Text.Encoding.GetEncoding("UTF-8"));
content = sr.ReadToEnd();
string resp = content.TrimStart('[').TrimEnd(']').TrimStart('"').TrimEnd('"');
if (resp == "\"The request is invalid.\"")
{
return "VALIDATE Me";
}
else
{
return resp;
}
}
It usually stops at the response variable then throws the exception from the method that calls this method, that there is no connection. I am thinking of making a while loop to make a countdown to reconnect for about an hour perhaps. Something like this:
int rt = 0;
while (rt < 60)
{
if (resp == "\"Unable to connect to the remote server.\"")
{
Console.Writeline("Connection Timed Out");
Console.Writeline("Re-establishing connection...");
DateTime startTime = DateTime.Now;
while (true)
{
if (DateTime.Now.Subtract(startTime).TotalMilliseconds > 60000)
break;
}
rt++;
Console.Writeline("Retrying " + rt.ToString() + " times");
}
if (rt >= 60)
{
Console.Writeline("Failed to reconnect.");
}
Any advise?
//this is by no means pretty, but im using your code verbatim
` public static string httpsGET(string passedweburi, string BCO)
{
string content = "";
//GET method
HttpWebRequest HttpRequest = (HttpWebRequest)WebRequest.Create(passedweburi + BCO);
HttpRequest.Method = "GET";
//Response
try
{
HttpWebResponse response = (HttpWebResponse)HttpRequest.GetResponse();
}
catch(Exception ex)
{
return "failed";
}
StreamReader sr = new StreamReader(response.GetResponseStream(), System.Text.Encoding.GetEncoding("UTF-8"));
content = sr.ReadToEnd();
string resp = content.TrimStart('[').TrimEnd(']').TrimStart('"').TrimEnd('"');
if (resp == "\"The request is invalid.\"")
{
return "VALIDATE Me";
}
else
{
return resp;
}
}
//calling your method
string resp = "";
while (rt < 60)
{
if (rt >= 60)
{
Console.Writeline("Failed to reconnect.");
}
resp = YourStaticObj.httpsGET("http://bla","bco")
if (resp == "failed")
{
Console.Writeline("Connection Timed Out");
Console.Writeline("Re-establishing connection...");
DateTime startTime = DateTime.Now;
System.Threading.Thread.Sleep(60000);
Console.Writeline("Retrying " + rt.ToString() + " times");
}
}
The following code gets stuck in a while loop when an exception is encountered. This is because GetResponse is caching the data.
According to fiddler, no data after the first request is issued.
Is it a best practice to recreate the webclient to solve the "refresh" issue I'm having?
private static ReportStatusEnum GetReportStatus(string domain, string oAuthKey, long permissionReportID)
{
string target = string.Format("https://{0}.egnyte.com/pubapi/v1/audit/jobs/{1}", domain, permissionReportID);
var client = new WebClient();
string result ="";
var request = (HttpWebRequest)WebRequest.Create(target);
request.ContentType = "application/json";
request.Headers.Add("Authorization", "Bearer " + oAuthKey);
request.AllowAutoRedirect = false;
bool callComplete = false;
while (callComplete != true)
{
try
{
using (var response = request.GetResponse())
{
using (StreamReader reader = new StreamReader(response.GetResponseStream()))
{
result = reader.ReadToEnd();
}
}
JToken result2 = JObject.Parse(result);
var statusResult = result2.SelectToken("status");
ReportStatusEnum ret = ReportStatusEnum.error;
Enum.TryParse<ReportStatusEnum>(statusResult.ToString(), out ret);
Console.WriteLine("The returned variable is:" + ret);
callComplete = true;
return ret;
}
catch (System.Net.WebException e)
{
if (e.Response != null)
if (e.Response.ContentLength > 0)
{
if (e.Response.Headers["X-Mashery-Error-Code"] == "ERR_403_DEVELOPER_OVER_QPS")
{
Thread.Sleep(60000); Console.Write("*QPS HIT*");
}
}
}
}
return ReportStatusEnum.error;
}
No. HttpWebRequests are not reusable.
Just move the creation of your WebRequest into the body of your loop:
string result ="";
bool callComplete = false;
while (callComplete != true)
{
var request = (HttpWebRequest)WebRequest.Create(target);
request.ContentType = "application/json";
request.Headers.Add("Authorization", "Bearer " + oAuthKey);
request.AllowAutoRedirect = false;
//...
Please see this thread:
In this thread we have the same issue answered about do...while. But what about foreach? Meaning how can we retry a try statement in try/catch inside this foreach with another proxy (another number of proxy_line_num integer):
foreach (string link_1 in links_with_kid)
{
try
{
getData = "";
req = (HttpWebRequest)WebRequest.Create(link_1);
req.Method = "GET";
req.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
req.UserAgent = "Mozilla/5.0 (Windows NT 6.1; rv:19.0) Gecko/20100101 Firefox/19.0";
req.ContentType = "text/html; charset=utf-8";
req.Referer = "http://www.example.com/";
req.KeepAlive = true;
req.Timeout = 25000;
if (useproxy)
{
string[] Ok_ip_port_ar = list_lines_GPL[proxy_line_num].Split(':');
proxy_line_num++;
if (proxy_line_num == list_lines_GPL.Count)
{
proxy_line_num = 0;
}
proxy = new WebProxy(Ok_ip_port_ar[0], int.Parse(Ok_ip_port_ar[1]));
}
req.Proxy = proxy;
req.CookieContainer = cookieJar1;
res = (HttpWebResponse)req.GetResponse();
Stream = res.GetResponseStream();
reader = new StreamReader(Stream);
reader_str = reader.ReadToEnd();
htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.LoadHtml(reader_str);
var images = from image in htmlDoc.DocumentNode.Descendants("img")
select image;
string address = string.Empty;
foreach (var image in images)
{
if (image.Attributes["src"] != null)
{
string[] address_ar = image.Attributes["src"].Value.Split('/');
string address_ar_last = address_ar[address_ar.Length - 1];
char[] address_ar_last_char = address_ar_last.ToCharArray();
if (address_ar_last_char.Length == 8
&&
Char.IsUpper(address_ar_last_char[0])
&&
Char.IsUpper(address_ar_last_char[1])
&&
Char.IsUpper(address_ar_last_char[2])
&&
Char.IsUpper(address_ar_last_char[3]))
{
address = image.Attributes["src"].Value;
string localFilename = #"c:\images-from-istgah\" + address_ar_last;
using (WebClient client = new WebClient())
{
client.DownloadFile(address, localFilename);
}
}
}
}
}
catch (Exception ex)
{
}
reader.Close();
Stream.Close();
res.Close();
}
I don't think that you can with a foreach. It is designed to give you the next item in the iterator.
If I were you I would use an ordinary for-loop with an iterator. That way you can control when to go to the next item.
Edit:
When writing it, a while actually made more sense in C#.
IEnumerator<String> iter = list.GetEnumerator();
bool bHasMore = iter.MoveNext();
while (bHasMore) {
try {
...
bHasMore = Iter.MoveNext();
}
catch ...
}
I dont have the entire refernce in my head, so you might need to look something up to get it to compile, but I hope not.