I have a a desktop application with a System.Windows.Form containing some TextBox controls. I need to validate the control values against restrictions of an xml schema.
For each TextBox I can retrieve the relevant XmlSchemaSimpleTypeRestriction from its type and then use a method as follows to validate its value:
public static bool Validate(XmlSchemaSimpleTypeRestriction restriction, string value)
{
bool isENum = false;
bool isValidEnum = false;
foreach (var item in restriction.Facets)
{
XmlSchemaLengthFacet lengthFacet = item as XmlSchemaLengthFacet;
if (lengthFacet != null)
{
int length = Int32.Parse(lengthFacet.Value);
if (!(value.Length == length))
return false;
}
XmlSchemaMinLengthFacet minLenghtFacet = item as XmlSchemaMinLengthFacet;
if (minLenghtFacet != null)
{
int length = Int32.Parse(minLenghtFacet.Value);
if (!(value.Length >= length))
return false;
}
XmlSchemaMaxLengthFacet maxLenghtFacet = item as XmlSchemaMaxLengthFacet;
if (maxLenghtFacet != null)
{
int length = Int32.Parse(maxLenghtFacet.Value);
if (!(value.Length <= length))
return false;
}
XmlSchemaPatternFacet patternFacet = item as XmlSchemaPatternFacet;
if (patternFacet != null)
{
Regex re = new Regex(patternFacet.Value);
if (!re.IsMatch(value))
return false;
}
XmlSchemaEnumerationFacet enumFacet = item as XmlSchemaEnumerationFacet;
if (patternFacet != null)
{
isENum = true;
if (StringComparer.InvariantCultureIgnoreCase.Compare(value, enumFacet.Value) == 0)
isValidEnum = true;
}
if (isENum && (!isValidEnum))
return false;
return true;
}
I am going to use this method in the Validating event of the controls. Is there any simpler way of doing this?
Ok, it's a little more complicated than I initially thought. Basically, you need create an XmlSchema that expects a single element with the provided restriction. Then you create an XML element with the provided value and validate it against the schema using an XmlReader:
public static bool Validate(XmlSchemaSimpleTypeRestriction restriction, string value)
{
var schema = new XmlSchema();
schema.Items.Add(new XmlSchemaElement
{
Name = "value",
SchemaType = new XmlSchemaSimpleType { Content = restriction }
});
var schemaSet = new XmlSchemaSet();
schemaSet.Add(schema);
var readerSettings = new XmlReaderSettings
{
ValidationType = ValidationType.Schema,
ValidationFlags = XmlSchemaValidationFlags.ReportValidationWarnings,
Schemas = schemaSet
};
string xml = new XElement("value", value).ToString();
try
{
var reader = XmlReader.Create(new StringReader(xml), readerSettings);
while (reader.Read()) ;
return true;
}
catch (XmlSchemaValidationException)
{
return false;
}
}
I tested it with this code:
static void Main(string[] args)
{
var restriction = new XmlSchemaSimpleTypeRestriction { BaseTypeName = new XmlQualifiedName("string", "http://www.w3.org/2001/XMLSchema") };
restriction.Facets.Add(new XmlSchemaMinLengthFacet { Value = "3" });
Console.WriteLine(Validate(restriction, "str"));
}
Related
I have below json data I need to apply CAdES-BES Signature with Automatic JSON Canonicalization. Please find my json data below. Helpful link from https://www.example-code.com/Csharp/itida_egypt_cades_bes_json_canonicalization.asp. I follow the steps but still digital signature is not applying. Its returns normal json data.
[HttpGet]
[Route("api/invoiceLines/")]
public IHttpActionResult getEInvoiceLines()
{
Chilkat.Crypt2 crypt = new Chilkat.Crypt2();
crypt.VerboseLogging = true;
Chilkat.Cert cert = new Chilkat.Cert();
cert.VerboseLogging = true;
// Set the smart card PIN, which will be needed for signing.
cert.SmartCardPin = "1245345";
// There are many ways to load the certificate.
// This example was created for a customer using an ePass2003 USB token.
// Assuming the USB token is the only source of a hardware-based private key..
bool success = cert.LoadFromSmartcard(#"E"); //Is this Right way To load certificate ?
Chilkat.JsonObject cmsOptions = new Chilkat.JsonObject();
// Setting "DigestData" causes OID 1.2.840.113549.1.7.5 (digestData) to be used.
cmsOptions.UpdateBool("DigestData", true);
cmsOptions.UpdateBool("OmitAlgorithmIdNull", true);
// Indicate that we are passing normal JSON and we want Chilkat do automatically
// do the ITIDA JSON canonicalization:
cmsOptions.UpdateBool("CanonicalizeITIDA", true);
crypt.CmsOptions = cmsOptions.Emit();
// The CadesEnabled property applies to all methods that create CMS/PKCS7 signatures.
// To create a CAdES-BES signature, set this property equal to true.
crypt.CadesEnabled = true;
crypt.HashAlgorithm = "sha256";
Chilkat.JsonObject jsonSigningAttrs = new Chilkat.JsonObject();
jsonSigningAttrs.UpdateInt("contentType", 1);
jsonSigningAttrs.UpdateInt("signingTime", 1);
jsonSigningAttrs.UpdateInt("messageDigest", 1);
jsonSigningAttrs.UpdateInt("signingCertificateV2", 1);
crypt.SigningAttributes = jsonSigningAttrs.Emit();
// By default, all the certs in the chain of authentication are included in the signature.
// If desired, we can choose to only include the signing certificate:
crypt.IncludeCertChain = false;
EInvoiceModel.Example ds = new EInvoiceModel.Example();
//Start issuer details
ds.issuer = new EInvoiceModel.Issuer();
ds.issuer.type = "B";
ds.issuer.id = "113317713";
ds.issuer.name = "Issuer Company";
//Start issuer address details
ds.issuer.address = new EInvoiceModel.Address();
ds.issuer.address.branchID = "1";
ds.issuer.address.country = "EG";
ds.issuer.address.governate = "Cairo";
ds.issuer.address.regionCity = "Nasr City";
ds.issuer.address.street = "stree1";
ds.issuer.address.buildingNumber = "Bldg. 0";
ds.issuer.address.postalCode = "68030";
ds.issuer.address.floor = "1";
ds.issuer.address.room = "123";
ds.issuer.address.landmark = "7660 Melody Trail";
ds.issuer.address.additionalInformation = "beside Town Hall";
//Start Receiver details
ds.receiver = new EInvoiceModel.Receiver();
ds.receiver.type = "B";
ds.receiver.id = "3125617";
ds.receiver.name = "Receiver company";
//Start Receiver address datails
ds.receiver.address = new EInvoiceModel.AddressReceiver();
ds.receiver.address.country = "EG";
ds.receiver.address.governate = "Cairo";
ds.receiver.address.regionCity = "Nasr City";
ds.receiver.address.street = "stree1";
ds.receiver.address.buildingNumber = "Bldg. 0";
ds.receiver.address.postalCode = "68030";
ds.receiver.address.floor = "1";
ds.receiver.address.room = "123";
ds.receiver.address.landmark = "7660 Melody Trail";
ds.receiver.address.additionalInformation = "beside Town Hall";
//Document type & version
ds.documentType = "i";
ds.documentTypeVersion = "1.0";
DateTime d = new DateTime();
ds.dateTimeIssued = d; //Invoice date
ds.taxpayerActivityCode = "9478"; //needed info
ds.internalID = "WADIn1234"; //Internal Invoice number
ds.salesOrderReference = "So1234"; //So number //optional
ds.salesOrderDescription = "SO1234"; //So additional Info //optional
ds.proformaInvoiceNumber = "SoPro123"; //optional
//Invoiceline Start
ds.invoiceLines = new List<EInvoiceModel.InvoiceLine>
{
new EInvoiceModel.InvoiceLine
{
description = "Computer1",
itemType = "GPC",
itemCode = "10001774",
unitType = "EA",
quantity = 2,
internalCode = "IC0",
salesTotal = 23.99,
total = 2969.89,
valueDifference = 7.00,
totalTaxableFees = 817.42,
netTotal = 880.71,
itemsDiscount = 5.00,
unitValue = new EInvoiceModel.UnitValue
{
currencySold = "EUR",
amountEGP = 189.40,
amountSold = 10.00,
currencyExchangeRate = 18.94,
},
discount = new EInvoiceModel.Discount
{
rate = 7,
amount = 66.29
},
taxableItems = new List<EInvoiceModel.TaxableItem>
{
new EInvoiceModel.TaxableItem
{
taxType = "T1",
amount = 272.07,
subType = "T1",
rate = 12
}
}
}
}; //Invoice Lines End
//Items total Discount and Sales/NetAmount
ds.totalDiscountAmount = 76.29;
ds.totalSalesAmount = 1609.90;
ds.netAmount = 1533.61;
//Tax Total Start
ds.taxTotals = new List<EInvoiceModel.TaxTotal>
{
new EInvoiceModel.TaxTotal
{
taxType = "T1",
amount = 477.54,
}
};//Tax Total End
//Total Sales Amount & discounts
ds.totalAmount = 5191.50;
ds.extraDiscountAmount = 5.00;
ds.totalItemsDiscountAmount = 14.00;
string strResultJson = JsonConvert.SerializeObject(ds);
//System.IO.File.WriteAllText(#"C:\inetpub\wwwroot\path.json", strResultJson);
// File.WriteAllText(#"ds.json", strResultJson);
// string jsonToSign = "{ ... }";
string jsonToSign = strResultJson;
// Create the CAdES-BES signature.
crypt.EncodingMode = "base64";
// Make sure we sign the utf-8 byte representation of the JSON string
crypt.Charset = "utf-8";
string sigBase64 = crypt.SignStringENC(jsonToSign);
// return Ok(ds);
return Ok(sigBase64);
}
public static string SerializeObject2(object obj, int indent = 0)
{
var sb = new StringBuilder();
if (obj != null)
{
string indentString = new string(' ', indent);
if (obj is string || obj.IsNumber())
{
sb.Append($"\"{obj}\"");
}
else if (obj.GetType().BaseType == typeof(Enum))
{
sb.Append($"\"{obj.ToString()}\"");
}
else if (obj is Array)
{
var elems = obj as IList;
sb.Append($"{indentString}- [{elems.Count}] :\n");
for (int i = 0; i < elems.Count; i++)
{
sb.Append(SerializeObject2(elems[i], indent + 4));
}
}
else
{
Type objType = (obj).GetType();
PropertyInfo[] props = objType.GetProperties();
foreach (PropertyInfo prop in props)
{
if (prop.GetIndexParameters().Length == 0)
{
object propValue = prop.GetValue(obj);
var elems = propValue as IList;
if (elems != null)
{
sb.Append($"\"{prop.Name.ToUpper()}\"");
foreach (var item in elems)
{
sb.Append($"\"{prop.Name.ToUpper()}\"");
sb.Append(SerializeObject2(item, indent + 4));
}
}
else
{
if (Assembly.GetExecutingAssembly().DefinedTypes.Select(p => p.Assembly).ToList().Contains(prop.PropertyType.Assembly))
{
sb.Append($"\"{prop.Name.ToUpper()}\"");
sb.Append(SerializeObject2(propValue, indent + 4));
}
else
{
sb.Append($"\"{prop.Name.ToUpper()}\"\"{propValue}\"");
}
}
}
else if (objType.GetProperty("Item") != null)
{
int count = -1;
if (objType.GetProperty("Count") != null &&
objType.GetProperty("Count").PropertyType == typeof(int))
{
count = (int)objType.GetProperty("Count").GetValue(obj, null);
}
for (int i = 0; i < count; i++)
{
object val = prop.GetValue(obj, new object[] { i });
sb.Append(SerializeObject2(val, indent + 4));
}
}
}
}
}
return sb.ToString();
}
I have a problem.
In my Android app I use a: Android.Support.V4.View.ViewPager to swap between a summary and a wallet. The summary page is filled with 3 TextViews and the Wallet is created with 1 GridView. Both the pages are getting filled with data from a HTTPS call from where the response will be parsed into a List. Now I want to refresh both the pages every second. So I tried this:
public void LoadOrderPage()
{
Android.Support.V4.View.ViewPager SummaryWalletSwitcher = FindViewById<Android.Support.V4.View.ViewPager>(Resource.Id.SummaryWalletSwitcher);
List<View> viewlist = new List<View>();
viewlist.Add(LayoutInflater.Inflate(Resource.Layout.AgentSummary, null, false));
viewlist.Add(LayoutInflater.Inflate(Resource.Layout.AgentWallet, null, false));
SummaryWalletAdapter ViewSwitchAdapter = new SummaryWalletAdapter(viewlist);
SummaryWalletSwitcher.Adapter = ViewSwitchAdapter;
Timer AgentInfo_Timer = new Timer();
AgentInfo_Timer.Interval = 1000;
AgentInfo_Timer.Elapsed += LoadAgentInfo;
AgentInfo_Timer.Enabled = true;
}
public void LoadAgentInfo(object sender, ElapsedEventArgs e)
{
TextView TextPortfolioValue = FindViewById<TextView>(Resource.Id.txtPortfolioValue);
TextView TextValueUSDT = FindViewById<TextView>(Resource.Id.txtValueUSDT);
TextView TextTotalValue = FindViewById<TextView>(Resource.Id.txtTotalValue);
GridView GridviewWallet = FindViewById<GridView>(Resource.Id.GridviewWallet);
if (FirstWalletRun == true)
{
List<wallet> EmptyWalletList = new List<wallet>();
WalletListAdapter = new WalletListAdapter(this, EmptyWalletList);
GridviewWallet.Adapter = WalletListAdapter;
FirstWalletRun = false;
}
PortfolioValue = 0;
ValueUSDT = 0;
TotalValue = 0;
string response = "";
AgentId = getSelectedAgentId();
if (AgentId == 0)
{
AgentId = 1;
}
try
{
WebClient client = new WebClient();
var reqparm = new System.Collections.Specialized.NameValueCollection();
reqparm.Add("agentid", AgentId.ToString());
reqparm.Add("devicetoken", DeviceToken);
byte[] responsebytes = client.UploadValues("https://www.test.nl/getagentwallet.php", "POST", reqparm);
IgnoreBadCertificates();
response = Encoding.UTF8.GetString(responsebytes);
response = response.Replace("\n", "").Replace("\t", "");
}
catch (Exception ex)
{
string exFullName = (ex.GetType().FullName);
string ExceptionString = (ex.GetBaseException().ToString());
TextPortfolioValue.Text = "Unknown";
TextValueUSDT.Text = "Unknown";
TextTotalValue.Text = "Unknown";
}
if (response != "No updates")
{
//Parse json content
var jObject = JObject.Parse(response);
//Create Array from everything inside Node:"Coins"
var walletPropery = jObject["Wallet"] as JArray;
//Create List to save Coin Data
walletList = new List<wallet>();
//Find every value in Array: coinPropery
foreach (var property in walletPropery)
{
//Convert every value in Array to string
var propertyList = JsonConvert.DeserializeObject<List<wallet>>(property.ToString());
//Add all strings to List
walletList.AddRange(propertyList);
}
//Get all the values from Name, and convert it to an Array
string[][] NamesArray = walletList.OrderBy(i => i.AgentId)
.Select(i => new string[] { i.AgentId.ToString(), i.Coin, i.Quantity.ToString(), i.AvgPrice.ToString(), i.Value.ToString() })
.Distinct()
.ToArray();
foreach (string[] str in NamesArray)
{
if (str[1] != "USDT")
{
PortfolioValue += decimal.Parse(str[4]);
}
else
{
ValueUSDT += decimal.Parse(str[4]);
}
}
TotalValue = PortfolioValue + ValueUSDT;
TextPortfolioValue.Text = Math.Round(PortfolioValue, 8).ToString();
TextValueUSDT.Text = Math.Round(ValueUSDT, 8).ToString();
TextTotalValue.Text = Math.Round(TotalValue, 8).ToString();
SortedWalletList = walletList.OrderBy(o => o.Coin).ToList();
if (WalletListAdapter == null)
{
//Fill the DataSource of the ListView with the Array of Names
WalletListAdapter = new WalletListAdapter(this, SortedWalletList);
GridviewWallet.Adapter = WalletListAdapter;
}
else
{
WalletListAdapter.refresh(SortedWalletList);
AgentInfoNeedsUpdate = true;
}
}
else
{
AgentInfoNeedsUpdate = false;
}
}
And in my WalletListAdapter I created the refresh function:
public void refresh(List<wallet> mItems)
{
this.mItems = mItems;
NotifyDataSetChanged();
}
But the GridviewWallet never get's filled or doesn't get shown. What am I doing wrong?
EDIT:
Maybe there is something wrong in the WalletListAdapter, so here is the code of the class:
public class WalletListAdapter : BaseAdapter<wallet>
{
public List<wallet> mItems;
private Context mContext;
public WalletListAdapter(Context context, List<wallet> items)
{
mItems = items;
mContext = context;
}
public override int Count
{
get { return mItems.Count; }
}
public void refresh(List<wallet> mItems)
{
this.mItems = mItems;
NotifyDataSetChanged();
}
public override long GetItemId(int position)
{
return position;
}
public override wallet this[int position]
{
get { return mItems[position]; }
}
public override View GetView(int position, View convertView, ViewGroup parent)
{
View row = convertView;
if (row == null)
{
row = LayoutInflater.From(mContext).Inflate(Resource.Layout.walletlist_row, null, false);
var txtWalletCoin = row.FindViewById<TextView>(Resource.Id.txtWalletCoin);
var txtWalletQuantity = row.FindViewById<TextView>(Resource.Id.txtWalletQuantity);
var txtAvgPrice = row.FindViewById<TextView>(Resource.Id.txtWalletAvgPrice);
var txtWalletValue = row.FindViewById<TextView>(Resource.Id.txtWalletValue);
var txtProfitUSDT = row.FindViewById<TextView>(Resource.Id.txtWalletProfitUSDT);
var txtProfitPerc = row.FindViewById<TextView>(Resource.Id.txtWalletProfitPerc);
row.Tag = new WalletViewHolder()
{
txtWalletCoin = txtWalletCoin,
txtWalletQuantity = txtWalletQuantity,
txtAvgPrice = txtAvgPrice,
txtWalletValue = txtWalletValue,
txtProfitUSDT = txtProfitUSDT,
txtProfitPerc = txtProfitPerc
};
}
var holder = (WalletViewHolder)row.Tag;
holder.txtWalletCoin.Text = mItems[position].Coin;
holder.txtWalletQuantity.Text = Math.Round(mItems[position].Quantity, 2).ToString();
holder.txtAvgPrice.Text = Math.Round(mItems[position].AvgPrice, 2).ToString();
holder.txtWalletValue.Text = Math.Round(mItems[position].Value, 2).ToString();
if (mItems[position].Coin != "USDT")
{
holder.txtProfitUSDT.Text = Math.Round(mItems[position].ProfitUSDT, 2).ToString();
holder.txtProfitPerc.Text = Math.Round(mItems[position].ProfitPerc, 1).ToString();
}
else
{
holder.txtProfitUSDT.Text = Math.Round(0.00, 2).ToString();
holder.txtProfitPerc.Text = Math.Round(0.00, 1).ToString();
}
return row;
}
}
Hope this discussion provides some insights to fix the gridview refresh using Xamarin. According to it, the grid has to be recreated.
https://forums.xamarin.com/discussion/115256/refresh-a-gridview
I need help to get data from the site. I use geckofx in my application. I want it to retrieve text data from the xpath location after loading the page
XPathResult xpathResult = geckoWebBrowser1.Document.EvaluateXPath("/html/body/table[3]/tbody/tr[1]/td[2]/a[1]");
IEnumerable<GeckoNode> foundNodes = xpathResult.GetNodes();
How to download data as text?
It looks like you are struggling to retrieve the text from the GeckoFX objects.
Here are a few calls and operations that should get you started:
//get by XPath
XPathResult xpathResult = _browser.Document.EvaluateXPath("//*[#id]/div/p[2]");
var foundNodes = xpathResult.GetNodes();
foreach (var node in foundNodes)
{
var x = node.TextContent; // get text text contained by this node (including children)
GeckoHtmlElement element = node as GeckoHtmlElement; //cast to access.. inner/outerHtml
string inner = element.InnerHtml;
string outer = element.OuterHtml;
//iterate child nodes
foreach (var child in node.ChildNodes)
{
}
}
//get by id
GeckoHtmlElement htmlElementById = _browser.Document.GetHtmlElementById("mw-content-text");
//get by tag
GeckoElementCollection byTag = _browser.Document.GetElementsByTagName("input");
foreach (var ele in byTag)
{
var y = ele.GetAttribute("value");
}
//get by class
var byClass = _browser.Document.GetElementsByClassName("input");
foreach (var node in byClass)
{
//...
}
//cast to a different object
var username = ((GeckoInputElement)_browser.Document.GetHtmlElementById("yourUsername")).Value;
//create new object from DomObject
var button = new GeckoButtonElement(_browser.Document.GetElementById("myBtn").DomObject);
public string extract(string xpath, string type)
{
string result = string.Empty;
GeckoHtmlElement elm = null;
GeckoWebBrowser wb = geckoWebBrowser1;//(GeckoWebBrowser)GetCurrentWB();
if (wb != null)
{
elm = GetElement(wb, xpath);
if (elm != null)
//UpdateUrlAbsolute(wb.Document, elm);
if (elm != null)
{
switch (type)
{
case "html":
result = elm.OuterHtml;
break;
case "text":
if (elm.GetType().Name == "GeckoTextAreaElement")
{
result = ((GeckoTextAreaElement)elm).Value;
}
else
{
result = elm.TextContent.Trim();
}
break;
case "value":
result = ((GeckoInputElement)elm).Value;
break;
default:
result = extractData(elm, type);
break;
}
}
}
return result;
}
private string extractData(GeckoHtmlElement ele, string attribute)
{
var result = string.Empty;
if (ele != null)
{
var tmp = ele.GetAttribute(attribute);
/*if (tmp == null)
{
tmp = extractData(ele.Parent, attribute);
}*/
if (tmp != null)
result = tmp.Trim();
}
return result;
}
private object GetCurrentWB()
{
if (tabControl1.SelectedTab != null)
{
if(tabControl1.SelectedTab.Controls.Count > 0)
//if (tabControl1.SelectedTab.Controls.Count > 0)
{
Control ctr = tabControl1.SelectedTab.Controls[0];
if (ctr != null)
{
return ctr as object;
}
}
}
return null;
}
private GeckoHtmlElement GetElement(GeckoWebBrowser wb, string xpath)
{
GeckoHtmlElement elm = null;
if (xpath.StartsWith("/"))
{
if (xpath.Contains("#class") || xpath.Contains("#data-type"))
{
var html = GetHtmlFromGeckoDocument(wb.Document);
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(html);
var node = doc.DocumentNode.SelectSingleNode(xpath);
if (node != null)
{
var currentXpath = "/" + node.XPath;
elm = (GeckoHtmlElement)wb.Document.EvaluateXPath(currentXpath).GetNodes().FirstOrDefault();
}
}
else
{
elm = (GeckoHtmlElement)wb.Document.EvaluateXPath(xpath).GetNodes().FirstOrDefault();
}
}
else
{
elm = (GeckoHtmlElement)wb.Document.GetElementById(xpath);
}
return elm;
}
private string GetHtmlFromGeckoDocument(GeckoDocument doc)
{
var result = string.Empty;
GeckoHtmlElement element = null;
var geckoDomElement = doc.DocumentElement;
if (geckoDomElement is GeckoHtmlElement)
{
element = (GeckoHtmlElement)geckoDomElement;
result = element.InnerHtml;
}
return result;
}
private void button5_Click(object sender, EventArgs e)
{
var text = extract("/html/body/table[3]/tbody/tr[1]/td[2]/a[2]", "text");
MessageBox.Show(text);
}
I also insert the code from which I used a little longer code but it also works. maybe someone will need it. The creator of the code is Đinh Công Thắng, Web Automation App,
Regards
Ok i am having a major problem atm.
My software is using extremely high amount of ram. I am using a lot of HtmlAgilityPack.HtmlDocument objects with big size pages sources.
However all of the objects are used inside static functions and HtmlAgilityPack.HtmlDocument isn't IDisposable
So do i need to set every variable explicitly to null ?
Even if they are inside static functions ?
For example do i need to set these variables to null at the end of the function below
The variables i am asking : lstDrwList ? Or since it is inside it will get disposed automatically ?
Should i call explicitly garbage collector ?
C# .net 4.5 WPF application
private static void func_CheckWaitingToProcessPages(Object state)
{
ParallelOptions myOptions = new ParallelOptions();
myOptions.MaxDegreeOfParallelism = PublicSettings.ir_How_Many_Tasks_For_Per_Pages_Process;
List<DataRow> lstDrwList = new List<DataRow>();
using (DataTable dtMyTable = DbConnection.db_Select_DataTable(srSelectTopProcessPagesQuery))
{
foreach (DataRow drw in dtMyTable.Rows)
{
lstDrwList.Add(drw);
}
}
Parallel.ForEach(lstDrwList, myOptions, drw =>
{
process_Given_Page(drw);
});
}
The problem is found issue is how to fix
Here the problem this happens in 10 seconds i used visual studio profiler
Here the full class that causes this huge memory leak issue
using HtmlAgilityPack;
using System;
using System.Collections.Generic;
using System.Data;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace doktora_tez_projesi_crawler_program
{
public static class PagesProcessor
{
private static Timer _timer;
private static int howManySeconds = 10;
public static void func_StartCrawlingWaitingUrls()
{
PublicStaticFunctions.AddMsgToEvents("Checking waiting to process crawled urls process started every " + howManySeconds + " seconds!");
_timer = new Timer(func_CheckWaitingToProcessPages, null, PublicSettings.irTimers_Delayed_Start_MiliSeconds, howManySeconds * 1000);
}
private static string srSelectTopProcessPagesQuery = " select top 100 cl_IdUrl,cl_RooSiteId,cl_CrawlSource,cl_CrawlOrgUrl from tblCrawlUrls " +
" where cl_PageProcessed=0 and cl_TotalCrawlTimes > 0 " +
" order by cl_LastProcessDate asc";
private static void func_CheckWaitingToProcessPages(Object state)
{
ParallelOptions myOptions = new ParallelOptions();
myOptions.MaxDegreeOfParallelism = PublicSettings.ir_How_Many_Tasks_For_Per_Pages_Process;
List<DataRow> lstDrwList = new List<DataRow>();
using (DataTable dtMyTable = DbConnection.db_Select_DataTable(srSelectTopProcessPagesQuery))
{
foreach (DataRow drw in dtMyTable.Rows)
{
lstDrwList.Add(drw);
}
}
Parallel.ForEach(lstDrwList, myOptions, drw =>
{
process_Given_Page(drw);
});
}
private class csProductFeatures
{
public string srProductRootSiteId = "null", srProductTitle = "null", srProductCode = "null", srProductImageLink = "null";
public string srProductDetailedExplanation = "null", srProductFeatures = "null", srCrawledOrgUrl = "null", srProductIdCode = "null";
public bool blPossibleProductPage = false, blFreeCargo = false, blProductPage = true;
public List<string> lstProductCategories = new List<string>();
public int irProductPrice = 0;
public List<csProductComments> lstProductComments = new List<csProductComments>();
public List<KeyValuePair<string, string>> lstProductFeatures = new List<KeyValuePair<string, string>>();
}
private class csProductComments
{
public string srCommentTitle = "null", srCommentPros = "null", srCommentCons = "null";
public int irCommentScore = 0; //0 = negative 5=full star
}
private static void process_Given_Page(DataRow drw)
{
csProductFeatures temp_ProductFeatures = new csProductFeatures();
temp_ProductFeatures.srProductRootSiteId = drw["cl_RooSiteId"].ToString();
temp_ProductFeatures.srCrawledOrgUrl = drw["cl_CrawlOrgUrl"].ToString();
HtmlDocument hdMyDoc = new HtmlDocument();//nulled
hdMyDoc.LoadHtml(drw["cl_CrawlSource"].ToString());
bool blBreakLoop = false;
foreach (var vrVariable in PublicVariables.dicRootSites[temp_ProductFeatures.srProductRootSiteId].lstRootSiteIdentifiers)
{
if (vrVariable.srHtmlObjectType != "link")
{
HtmlNodeCollection hdNodes;
if (vrVariable.blSelectMultipleNodes == false)
hdNodes = hdMyDoc.DocumentNode.SelectNodes(string.Format("//{0}[#{1}='{2}']", vrVariable.srHtmlObjectType,
vrVariable.srHtmlObjectTypeIdentifier, vrVariable.srHtmlObjectTypeName));
else
hdNodes = hdMyDoc.DocumentNode.SelectNodes(string.Format("//{0}[#{1}='{2}']//{3}", vrVariable.srHtmlObjectType,
vrVariable.srHtmlObjectTypeIdentifier, vrVariable.srHtmlObjectTypeName, vrVariable.srHtmlSubIdentifierType));
if (hdNodes == null && vrVariable.srIndetifierType == "ProductTitle")
{
blBreakLoop = true;
temp_ProductFeatures.blProductPage = false;
continue;
}
if (blBreakLoop == true)
break;
if (hdNodes == null)
continue;
string sr_Node_Required_Val = "null";
if (hdNodes[0].InnerText != null)
sr_Node_Required_Val = hdNodes[0].InnerText;
string srLinkVal = "null";
if (vrVariable.srHtmlObjectType == "a" && hdNodes[0].Attributes != null)
{
if (hdNodes[0].Attributes["href"] != null)
{
srLinkVal = PublicStaticFunctions.Return_Absolute_Url(hdNodes[0].Attributes["href"].Value, temp_ProductFeatures.srCrawledOrgUrl);
}
}
if (vrVariable.blGetValue == true)
{
if (hdNodes[0].Attributes != null)
if (hdNodes[0].Attributes["value"] != null)
sr_Node_Required_Val = hdNodes[0].Attributes["value"].Value;
}
sr_Node_Required_Val = sr_Node_Required_Val.Trim();
switch (vrVariable.srIndetifierType)
{
case "ProductPage":
temp_ProductFeatures.blPossibleProductPage = true;
break;
case "ProductTitle":
temp_ProductFeatures.srProductTitle = sr_Node_Required_Val;
break;
case "ProductCode":
temp_ProductFeatures.srProductCode = sr_Node_Required_Val;
break;
case "ProductCargo":
temp_ProductFeatures.blFreeCargo = true;
break;
case "ProductCategories":
temp_ProductFeatures.lstProductCategories = func_Return_Product_Categories(hdNodes);
break;
case "ProductPrice":
temp_ProductFeatures.irProductPrice = func_Return_Product_Price(sr_Node_Required_Val, temp_ProductFeatures.srProductRootSiteId);
break;
case "ProductImage":
temp_ProductFeatures.srProductImageLink = srLinkVal;
break;
case "ProductIdCode":
temp_ProductFeatures.srProductIdCode = sr_Node_Required_Val;
break;
}
}
if (vrVariable.srHtmlObjectType == "link")
{
string srLinkToFetch = vrVariable.srHtmlObjectTypeIdentifier;
if (vrVariable.blUsesProductIdCode == true)
{
srLinkToFetch = string.Format(srLinkToFetch, temp_ProductFeatures.srProductIdCode);
}
string srFetchResult = CrawlGivenUrl.func_fetch_Page(srLinkToFetch);
string srResultToAssign = "null";
if (srFetchResult == PublicSettings.srCrawlFailedMessage)
{
srResultToAssign = srFetchResult;
}
else
{
HtmlDocument temp_HdDocument = new HtmlDocument();//nulled
temp_HdDocument.LoadHtml(srFetchResult);
if (temp_HdDocument.DocumentNode != null)
if (temp_HdDocument.DocumentNode.InnerText != null)
srResultToAssign = temp_HdDocument.DocumentNode.InnerText;
temp_HdDocument = null;
}
switch (vrVariable.srIndetifierType)
{
case "ProductExplanation":
temp_ProductFeatures.srProductDetailedExplanation = srResultToAssign;
break;
case "ProductFeatures":
temp_ProductFeatures.lstProductFeatures = func_Return_Product_Features(temp_ProductFeatures.srProductRootSiteId, srFetchResult, temp_ProductFeatures.srCrawledOrgUrl);
break;
}
}
}
if (temp_ProductFeatures.blProductPage == true)
{
string asdas = "";
}
hdMyDoc = null;
}
private static List<string> func_Return_Product_Categories(HtmlNodeCollection hdNodeCollection)
{
List<string> lstCategories = new List<string> { };
foreach (HtmlNode hdNode in hdNodeCollection)
{
if (hdNode.InnerText != null)
{
lstCategories.Add(hdNode.InnerText);
}
}
return lstCategories;
}
private static int func_Return_Product_Price(string srPriceText, string srRootSiteId)
{
int irPrice = 0;
srPriceText = srPriceText.Replace(PublicVariables.dicRootSites[srRootSiteId].srPriceDelimeter, "");
if (srPriceText.Contains(PublicVariables.dicRootSites[srRootSiteId].srPriceIgnoreDelimeter) == true)
{
srPriceText = srPriceText.Substring(0, srPriceText.IndexOf(PublicVariables.dicRootSites[srRootSiteId].srPriceIgnoreDelimeter));
}
Int32.TryParse(srPriceText, out irPrice);
return irPrice;
}
private static List<KeyValuePair<string, string>> func_Return_Product_Features(string srRootSiteId, string srPageSource, string srCrawlUrl)
{
List<KeyValuePair<string, string>> lstFoundFeatures = new List<KeyValuePair<string, string>>();
if (srPageSource == PublicSettings.srCrawlFailedMessage)
return lstFoundFeatures;
HtmlDocument temp_HdDocument = new HtmlDocument();//nulled
temp_HdDocument.LoadHtml(srPageSource);
List<string> lstFeatureTitles = new List<string>();
List<string> lstFeatureDescriptions = new List<string>();
foreach (var vrVariable in PublicVariables.dicRootSites[srRootSiteId].lstRootSitesFeaturesIdentifiers)
{
if (vrVariable.blPerFeatureIdentifier == true)
{
HtmlNodeCollection hdNodes = temp_HdDocument.DocumentNode.SelectNodes(string.Format("//{0}[#{1}='{2}']", vrVariable.srHtmlObjectType,
vrVariable.srHtmlObjectIdentifier, vrVariable.srHtmlObjectIdentifierName));
if (hdNodes != null)
foreach (var vrNewVariable in PublicVariables.dicRootSites[srRootSiteId].lstRootSitesFeaturesIdentifiers)
{
if (vrNewVariable.blPerFeatureIdentifier == false)
{
foreach (HtmlNode hdTempNode in hdNodes)
{
var vrTempNewNode = hdTempNode.SelectSingleNode(string.Format("//{0}[#{1}='{2}']", vrVariable.srHtmlObjectType,
vrVariable.srHtmlObjectIdentifier, vrVariable.srHtmlObjectIdentifierName));
if (vrTempNewNode != null)
if (vrTempNewNode.InnerText != null)
{
string srNodeFeature = vrTempNewNode.InnerText.Trim();
switch (vrVariable.srWhichFeatureIdentifier)
{
case "FeatureTitle":
lstFeatureTitles.Add(srNodeFeature);
break;
case "FeatureDescription":
lstFeatureDescriptions.Add(srNodeFeature);
break;
}
}
}
}
}
break;
}
}
temp_HdDocument = null;
if (lstFeatureDescriptions.Count != lstFeatureTitles.Count)
{
ErrorLogger.LogError("found features count not equal to features description count crawled url: " + srCrawlUrl);
return lstFoundFeatures;
}
for (int i = 0; i < lstFeatureDescriptions.Count; i++)
{
KeyValuePair<string, string> myKeyValPair = new KeyValuePair<string, string>(lstFeatureTitles[i], lstFeatureDescriptions[i]);
lstFoundFeatures.Add(myKeyValPair);
}
return lstFoundFeatures;
}
}
}
No, you don't need to set variables to null in both static and instance methods. The variables inside a method (even inside static method) are on the stack space of the method, so generally they will go out of scope at the end of method execution and will be targeted for garbage collection. And Generally explicitly calling the garbage collector isn't a good practice.
Here's my ICommand:
public ICommand ConfirmLotSavedCommand {
get
{
return new RelayCommand(ConfirmLotSaved);
}
}
The problem is I have deserialized data that I want to store into database after a user clicks confirm button. If the user does not click on confirm or the lot number already exists, then I don't want to save the deserialized string in db.
I had trouble calling a function with one parameter inside my ConfirmLotSaved() method because of scope.
So I created a set the deserialized lot as a field and put the code to save to db inside of ConfirmLotSaved(). However, the field is null for some strange reason... I'm not sure why.
Here's my attempt:
private LotInformation lot; //field that is supposed to contain all the deserialized info
private void ConfirmLotSaved()
{
using (var db = new DDataContext())
{
bool lotNumDbExists = db.LotInformation.Any(r => r.lot_number == DeserialLotNumber);
if (lotNumDbExists == false)
{
successWindow.Message = "Successfully Saved Lot";
dialogService.ShowDialog(successWindow.Message, successWindow);
LotInformation newLot = new LotInformation();
if (newLot != null)
{
newLot.Id = lot.Id;
newLot.lot_number = lot.lot_number;
newLot.exp_date = lot.exp_date;
LotNumber = Lot.lot_number;
ExpirationDate = Lot.exp_date.ToString();
foreach (Components comp in lot.Components)
{
newLot.Components.Add(comp);
}
ComponentsList = newLot.Components;
foreach (Families fam in lot.Families)
{
newLot.Families.Add(fam);
}
FamiliesList = newLot.Families;
try
{
db.LotInformation.Add(newLot);
db.SaveChanges();
//Grabs the lot_number column from db that is distinct
var lotNum = db.LotInformation.GroupBy(i => i.lot_number).Select(group => group.FirstOrDefault());
//Loops through the lot numbers column in db and converts to list
foreach (var item in lotNum)
{
Console.WriteLine(item.lot_number);
}
LotNumList = lotNum.ToList();
Console.WriteLine("successfully");
}
catch
{
//TODO: Add a Dialog Here
}
}
else if (lotNumDbExists == true)
{
// Inform user that the lot_number already exists
errorWindow.Message = LanguageResources.Resource.Lot_Exists_Already;
dialogService.ShowDialog(LanguageResources.Resource.Error, errorWindow);
logger.writeErrLog(LanguageResources.Resource.Lot_Exists_Already);
return;
}
}
}
}
Deserialization function to see where lot is grabbing data:
public void DeserializedStream(string filePath)
{
XmlRootAttribute xRoot = new XmlRootAttribute();
xRoot.ElementName = "lot_information";
xRoot.IsNullable = false;
// Create an instance of lotinformation class.
LotInformation lot = new LotInformation();
// Create an instance of stream writer.
TextReader txtReader = new StreamReader(filePath);
// Create and instance of XmlSerializer class.
XmlSerializer xmlSerializer = new XmlSerializer(typeof(LotInformation), xRoot);
// DeSerialize from the StreamReader
lot = (LotInformation)xmlSerializer.Deserialize(txtReader);
// Close the stream reader
txtReader.Close();
LotInformation newList = new LotInformation();
using (var db = new DDataContext())
{
bool isDuplicate = db.LotInformation.Any(r => r.lot_number == lot.lot_number);
if (newList != null && isDuplicate == false)
{
newList.Id = lot.Id;
newList.lot_number = lot.lot_number;
newList.exp_date = lot.exp_date;
DeserialLotNumber = newList.lot_number;
DeserialExpirationDate = newList.exp_date.ToString();
foreach (Component comp in lot.Components)
{
newList.Components.Add(comp);
}
DeserialComponentsList = newList.Components;
foreach (Families fam in lot.Families)
{
newList.Families.Add(fam);
}
DeserialFamiliesList = newList.Families;
}
else if (isDuplicate == true)
{
DeserialAnalytesList = null;
DeserialFamiliesList = null;
// Inform user that the lot_number already exists
errorWindow.Message = LanguageResources.Resource.Lot_Exists_Already;
dialogService.ShowDialog(LanguageResources.Resource.Error, errorWindow);
logger.writeErrLog(LanguageResources.Resource.Lot_Exists_Already);
return;
}
}
}
I figured out what was wrong:
After setting private LotInformation lot; field before constructor, I redeclared locally my mistake:
LotInformation lot = new LotInformation();
Changed it to:
lot = new LotInformation();
and it works.
I suggest you to use RelayCommand's generic edition http://www.kellydun.com/wpf-relaycommand-with-parameter/
It will allow you to pass lot to your command from view, all you need to store lot in current DataContext.