We have a recursive function we are using to bulk insert records into Cosmos.
While running a long migration to insert many records it appears that some records failed to insert but we have been unable to see why from the logging we have.
My assumption is a certain type of status code is not being retried when it should be or something is failing silently.
Can anyone advise what might be causing records to fail without being logged/being retried?
AllowBulkExecution = true is set when creating the container that is passed in to the code below.
public async Task<bool> TryBulkInsertAsync<T>(List<T> audits, int retryAttempts, TimeSpan retryDelay, int currentAttempt = 0, Container container = null) where T : BaseCosmosModel
{
if (currentAttempt > retryAttempts)
{
_logger.LogError($"Failed number of max retries ${retryAttempts}");
return false;
}
if (container == null)
{
container = _cosmosContainerFactory.BuildCosmosContainer<T>();
}
var attemptAudits = audits.Select(a => new CosmosInsertAttempt<T>
{
Audit = a,
RunningTask = container.CreateItemAsync(a)
}).ToList();
try
{
await Task.WhenAll(attemptAudits.Select(a => a.RunningTask));
var failedAudits = attemptAudits.Where(a =>
a.RunningTask.Result.StatusCode == HttpStatusCode.TooManyRequests ||
a.RunningTask.Result.StatusCode == HttpStatusCode.PreconditionFailed ||
a.RunningTask.Result.StatusCode == HttpStatusCode.RequestTimeout ||
a.RunningTask.Result.StatusCode == HttpStatusCode.ServiceUnavailable
).Select(a => a.Audit).ToList();
var nonRetryableAudits = attemptAudits.Where(a =>
!failedAudits.Contains(a.Audit) && (a.RunningTask.Result.StatusCode < (HttpStatusCode) 200 ||
a.RunningTask.Result.StatusCode > (HttpStatusCode) 299));
foreach (var audit in nonRetryableAudits)
{
_logger.LogError("Audit failed to bulk insert with non-retryable status code {cosmosAuditInsert}", audit.Audit);
}
if (failedAudits.Count > 0)
{
_logger.LogError("Retrying bulk insert from incorrect status code but no error, count: {retrySize}", failedAudits.Count);
return await TryBulkInsertAsync(failedAudits, retryAttempts, retryDelay, currentAttempt + 1, container);
}
return true;
}
catch (Exception)
{
await Task.Delay(retryDelay);
var failedAuditsWithException = attemptAudits
.Where(a => a.RunningTask.Exception != null)
.Select(a => a.Audit).ToList();
var failedAuditsWithBadStatusCode = attemptAudits.Where(a =>
(
a.RunningTask.Exception == null && a.RunningTask.Result != null &&
(a.RunningTask.Result.StatusCode == HttpStatusCode.TooManyRequests ||
a.RunningTask.Result.StatusCode == HttpStatusCode.PreconditionFailed ||
a.RunningTask.Result.StatusCode == HttpStatusCode.RequestTimeout ||
a.RunningTask.Result.StatusCode == HttpStatusCode.ServiceUnavailable)
)).Select(a => a.Audit).ToList();
if (failedAuditsWithBadStatusCode.Any())
{
_logger.LogError("Retrying bulk insert from incorrect status code but no error (some have exceptions), count: {retrySize}", failedAuditsWithBadStatusCode.Count);
}
var failedAudits = failedAuditsWithException.Concat(failedAuditsWithBadStatusCode).ToList();
var nonRetryableAudits = attemptAudits.Where(a =>
!failedAudits.Contains(a.Audit) && a.RunningTask.Exception == null && (a.RunningTask.Result.StatusCode < (HttpStatusCode)200 ||
a.RunningTask.Result.StatusCode > (HttpStatusCode)299));
foreach (var audit in nonRetryableAudits)
{
_logger.LogError("Audit failed to bulk insert with non-retryable status code {cosmosAuditInsert}", audit.Audit);
}
if (failedAudits.Count > 0)
{
_logger.LogError("Retrying bulk insert {failedAuditCount}", failedAudits.Count);
return await TryBulkInsertAsync(failedAudits, retryAttempts, retryDelay, currentAttempt + 1, container);
}
return true;
}
}
Related
I have to two function which is used to find tags inside the tags like, there is a tag A=B(C(D(E))) so i have to find all the tags inside B then all the tags inside C and so on. I write two function but getting the error System.StackOverflowException. In the first function i am providing the tag ID and against that tag id i am getting getNestesCalTagsId and then calling the getNestedCalTagsIngredients() function. But when there are lot of recursion calls i get the error System.StackOverflowException. Below is my whole code.
public List<int?> getNestedCalTags(int? calTagId)
{
var getNestesCalTagsId = db.Dependencies_Metrix.Where(x => x.Cal_Tag_P_Id == calTagId && x.Status == true && x.Cal_Tag_Id_FK!=null).Select(x => x.Cal_Tag_Id_FK).ToList();
if (getNestesCalTagsId.Count > 0)
{
nestedCalFTags.AddRange(getNestesCalTagsId);
foreach (var item in getNestesCalTagsId)
{
if (item != null)
{
getNestedCalTagsIngredients(item.Value);
}
}
}
if (nestedCalFTags.Count > 0)
{
int countedTags = nestedCalFTags.Count;
List<int?> tags = new List<int?>(nestedCalFTags);
for (int i = 0; i < tags.Count; i++)
{
if (tags[i] != null)
{
getNestedCalTagsIngredients(tags[i].Value);
}
}
}
return nestedRawTags;
}
public bool getNestedCalTagsIngredients(int nestCalTagId)
{
var getCalTags = db.Dependencies_Metrix.Where(x => x.Cal_Tag_P_Id == nestCalTagId && x.Status == true).ToList();
if (getCalTags.Count > 0)
{
foreach (var item in getCalTags)
{
if (item.Cal_Tag_Id_FK != null)
{
var getNestedCalTagParent = db.Dependencies_Metrix.Where(x => x.Cal_Tag_P_Id == item.Cal_Tag_Id_FK && x.Status == true && x.Cal_Tag_Id_FK!=null).Select(x => x.Cal_Tag_Id_FK).ToList();
if (getNestedCalTagParent != null)
{
nestedCalFTags.AddRange(getNestedCalTagParent);
getNestedCalTags(item.Cal_Tag_Id_FK);
}
}
else
{
var rawTagId = db.Dependencies_Metrix.Where(x => x.Cal_Tag_P_Id == item.Cal_Tag_P_Id && x.Real_Tag_Id_FK!=null).Select(x => x.Real_Tag_Id_FK).ToList();
if (rawTagId != null)
{
foreach (var rawItem in rawTagId)
{
if (rawItem!=null)
{
if (nestedRawTags.IndexOf(rawItem.Value) == -1)
{
nestedRawTags.Add(rawItem.Value);
}
}
}
}
nestedCalFTags.Remove(nestCalTagId);
}
}
}
return true;
}
Trying to delete nodes under certain conditions. Basically if certain checkboxes are checked I give an extra query with WHERE statement to my IENumerable named upit. After the queries has been set im trying to delete them iterating through every one, but nothing gets deleted everytime.
XDocument X = XDocument.Load(#"Financije.xml");
var upit = X.Element("POPIS").Elements("PODACI");
if (mjesec.Checked) { upit = upit.Where(E => (Convert.ToInt32(E.Element("MJESEC").Value) == Convert.ToInt32(mjesecbox.Text))); }
if (godina.Checked) { upit = upit.Where(E => (Convert.ToInt32(E.Element("GODINA").Value) == Convert.ToInt32(godinabox.Text))); }
if (ime.Checked) { upit = upit.Where(E => (E.Element("IME").Value.ToString().ToLower().Contains(search.Text.ToString().ToLower()))); }
if (opis.Checked) { upit = upit.Where(E => (E.Element("OPIS").Value.ToString().ToLower().Contains(search.Text.ToString().ToLower()))); }
if (veceod.Checked) { upit = upit.Where(E => (Convert.ToInt32(E.Element("CIJENA").Value.ToString()) > Convert.ToInt32(iznos.Text.ToString()))); }
if (manjeod.Checked) { upit = upit.Where(E => (Convert.ToInt32(E.Element("CIJENA").Value.ToString()) < Convert.ToInt32(iznos.Text.ToString()))); }
foreach (var item in upit)
{
upit.Remove();
}
and this is my XML file
<?xml version="1.0" encoding="utf-8"?>
<POPIS>
<PODACI>
<IME>test</IME>
<CIJENA>200</CIJENA>
<DATUM>12.1.2019</DATUM>
<MJESEC>1</MJESEC>
<GODINA>2019</GODINA>
<OPIS>test123333</OPIS>
</PODACI>
<PODACI>
<IME>voda</IME>
<CIJENA>230</CIJENA>
<DATUM>12.4.2018</DATUM>
<MJESEC>4</MJESEC>
<GODINA>2018</GODINA>
<OPIS>yes123no</OPIS>
</PODACI>
<PODACI>
<IME>oops</IME>
<OPIS>nice</OPIS>
<CIJENA>3</CIJENA>
<MJESEC>5</MJESEC>
<GODINA>2018<GODINA/>
<DATUM>24.02.2019</DATUM>
</PODACI>
<PODACI>
<IME>test</IME>
<OPIS>123</OPIS>
<CIJENA>1</CIJENA>
<MJESEC>12</MJESEC>
<GODINA>2019<GODINA/>
<DATUM>24.02.2019</DATUM>
</PODACI>
</POPIS>
It's a bit unclear what you want to end up with, but I assume you want to exclude items where they fail at least one case where the checkbox is checked and that comparison you are doing fails. So below achieves it in one query:
var X = XDocument.Load("Financije.xml");
var upit = X.Element("POPIS").Elements("PODACI");
var toRemove = upit.Where(u =>
// If mjesec is not checked, skip the predicate, otherwise evaluate it.
(!mjesec.Checked || (Convert.ToInt32(u.Element("MJESEC").Value) == Convert.ToInt32(mjesecbox.Text)))
// *And* do the same for godina and the rest of checkboxes...
&& (!godina.Checked || (Convert.ToInt32(u.Element("GODINA").Value) == Convert.ToInt32(godinabox.Text)))
&& (!ime.Checked || (u.Element("IME").Value.ToString().ToLower().Contains(search.Text.ToString().ToLower())))
&& (!opis.Checked || (u.Element("OPIS").Value.ToString().ToLower().Contains(search.Text.ToString().ToLower())))
&& (!veceod.Checked || (Convert.ToInt32(u.Element("CIJENA").Value.ToString()) > Convert.ToInt32(iznos.Text.ToString())))
&& (!manjeod.Checked || (Convert.ToInt32(u.Element("CIJENA").Value.ToString()) < Convert.ToInt32(iznos.Text.ToString()))));
X.Element("POPIS").ReplaceAll(upit.Except(toRemove));
X.Save("Financije.xml");
Give it a try
XDocument X = XDocument.Load(#"Financije.xml");
var upit = X.Element("POPIS").Elements("PODACI");
//I‘m only on phone so please excuse, if I take the wrong Type. Assign to null, to prevent Compiler-Error
IEnumerable<XElement> upitPart= null;
if (mjesec.Checked) { upitPart = upit.Where(E => (Convert.ToInt32(E.Element("MJESEC").Value) == Convert.ToInt32(mjesecbox.Text))); }
if (godina.Checked) { upitPart = upitPart.Where(E => (Convert.ToInt32(E.Element("GODINA").Value) == Convert.ToInt32(godinabox.Text))); }
if (ime.Checked) { upitPart = upitPart.Where(E => (E.Element("IME").Value.ToString().ToLower().Contains(search.Text.ToString().ToLower()))); }
if (opis.Checked) { upitPart = upitPart.Where(E => (E.Element("OPIS").Value.ToString().ToLower().Contains(search.Text.ToString().ToLower()))); }
if (veceod.Checked) { upitPart = upitPart.Where(E => (Convert.ToInt32(E.Element("CIJENA").Value.ToString()) > Convert.ToInt32(iznos.Text.ToString()))); }
if (manjeod.Checked) { upitPart = upitPart.Where(E => (Convert.ToInt32(E.Element("CIJENA").Value.ToString()) < Convert.ToInt32(iznos.Text.ToString()))); }
if(upitPart != null)
{
foreach (var item in upitPart)
{
upit.Remove(item);
}
}
Just trying to get a cleaner code on a delete method. I need to delete records from a database if a certain column value matches one of two columns in another table.
Is there a better way to delete multiple records, with a "OR"-like expression, so that I can have only one for each loop instead of the following two?
public static void DeleteStageById(int StageId, int ApplicationId)
{
using (IPEntities ip = IPEntities.New())
{
var stage = ip.mkStages;
var stageCultures = ip.appObjectCultures;
var stageStates = ip.mkStatesInStages;
foreach (var stageCulture in stageCultures.Where(sC => sC.ObjectCultureId == stage.Where(s => s.StageId == StageId && s.ApplicationId == ApplicationId).FirstOrDefault().OCId_Name))
{
stageCultures.DeleteObject(stageCulture);
}
foreach (var stageCulture in stageCultures.Where(sC => sC.ObjectCultureId == stage.Where(s => s.StageId == StageId && s.ApplicationId == ApplicationId).FirstOrDefault().OCId_Description))
{
stageCultures.DeleteObject(stageCulture);
}
...
ip.SaveChanges();
}
}
my linq would look like this one
var stage = ip.mkStages;
var stageCultures = ip.appObjectCultures;
var stageStates = ip.mkStatesInStages;
//store this result into a temp variable so it only needs to run once
var temp = stage.Where(s => s.StageId == StageId && s.ApplicationId == ApplicationId).FirstOrDefault();
if (temp != null)
{
foreach (var stageCulture in stageCultures.Where(sC => sC.ObjectCultureId == temp.OCId_Name || sC.ObjectCultureId == temp.OCId_Description))
{
stageCultures.DeleteObject(stageCulture);
}
...
ip.SaveChanges();
}
I recommend avoiding confusing expressions, but here you go:
foreach (var stageCulture in stageCultures.Where(sC => {
var v = stage.Where(s => s.StageId == StageId && s.ApplicationId == ApplicationId).FirstOrDefault();
return sC.ObjectCultureId == v.OCId_Name || sC.ObjectCultureId == v.OCId_Description;
})
{
stageCultures.DeleteObject(stageCulture);
}
As far as i understand, this exception is thrown because multiple of my threads try to access the same data on the database. But the exception is thrown in a line where i convert an enumerable to a list.
Can anyone explain this?
(The Program.GetContext(), is used to get the current threads context to the database, since it's multithreaded, and uses EF this is a way to do it)
var loadedSammenSat = Program.GetContext().SammensatKodetVaerdier.Where(o => (o.Kode == kode) && (o.Kodetekst == kodetekst) && (o.Forkortelse == forkortelse) && (o.Navn == navn)).Include(b => b.Tillaegskode).ToList();
if (!sammensatKodet.Tillaegskode.Any())
{
var selceted = loadedSammenSat.FirstOrDefault(o => (o.Tillaegskode == null || o.Tillaegskode.Count == 0));
if (selceted != null)
return selceted;
}
else
{
var selcetedWithOutList = loadedSammenSat.Where(o => (o.Tillaegskode != null && o.Tillaegskode.Count() == sammensatKodet.Tillaegskode.Count()));
var selceted = selcetedWithOutList.ToList(); //LINE THAT THROW EXCEPTION!!
if (selceted.Count() > 0)
{
List<DBTillaegsKode> list = new List<DBTillaegsKode>();
sammensatKodet.Tillaegskode.ForEach(p => list.Add(new DBTillaegsKode(p.KodetVaerdi)));
foreach (var dbSammensatKodetVaerdi in selceted)
{
var mismatch = dbSammensatKodetVaerdi.Tillaegskode.Except(list);
if(mismatch.GetEnumerator().Current != null)
if (mismatch.Count() == 0)
{
return dbSammensatKodetVaerdi;
}
}
}
}
LINQ is lazy-loaded - it's only when you do something with the resultset (in this case doing a .ToList() operation) that the query is executed.
I have a LINQ statement handling the Login process. It works fine when passed valid username and password combinations. However when I test on invalid credentials, I get a NullReferenceException error on the line below indicated by <<<---------------- Need some help on proper handling of invalid credentials please?
public int _accountID;
public int _securityLevelID;
public void GetLoginInfo(string EmailAddress, string Password)
{
LoginItem l = null;
{
try
{
using (RootsDataContext RDC = new RootsDataContext())
l = (from a in RDC.DBLogIns
where a.EmailAddress == EmailAddress
&& a.Password == Password
&& a.IsActive == 1
select new LoginItem
{
AccountIDFK = a.AccountIDFK,
SecurityLevelIDFK = a.SecurtityLevelIDFK,
}).FirstOrDefault();
_accountID = (int)l.AccountIDFK; <<<----------------
_securityLevelID = (int)l.SecurityLevelIDFK;
if (_accountID < 1 || _accountID == null)
{
lbl_LoginStatus.Text = "Invalid";
}
}
catch (Exception ex)
{
string error = ex.Message;
}
if (_accountID > 0)
{
if (_accountID == 1 && _securityLevelID == 1) // [Quentin]
{
Response.Redirect("~/AccountsMaster.aspx");
}
if (_accountID > 1 && _securityLevelID == 2) // [Companies]
{
Response.Redirect("~/CompanyMaster.aspx");
}
if (_accountID > 1 && _securityLevelID == 3) // [Branch]
{
Response.Redirect("~/BranchMaster.Aspx");
}
if (_accountID > 1 && _securityLevelID == 4) // [Clients]
{
Response.Redirect("~/Home.aspx");
}
}
}
}
By saying
// ...
}).FirstOrDefault();
You will either get an DBLogIn object if a match is found, or null if it is not.
You will need to check for null before accessing the property AccountIDFK and SecurityLevelIDFK:
// ... }).FirstOrDefault();
if (l != null)
{
_accountID = (int)l.AccountIDFK;
_securityLevelID = (int)l.SecurityLevelIDFK;
}
Some other points to consider:
You shouldn't store passwords directly in the database. A more secure approach is to store hashed (and potentially salted) passwords in the database, and then to find the user (by EmailAddress and Active = 1), and then compare the hashes of what the user typed, and what is stored in the DB.
This code swallows exceptions - this makes diagnosing problems a nightmare:
catch (Exception ex)
{
string error = ex.Message;
}
Don't make fields public (public int _accountID;) - Make them private if they are not used externally, or convert them to (autogenerated) Properties if they are externally visible from your class.
FirstOrDefault method will return null if there's no DBLogIn records match the criteria you give, so you need to check if l is null first before accessing (int)l.AccountIDFK. Moreover, it looks like lbl_LoginStatus.Text = "Invalid"; should be done when l is null, so you need to remove if (_accountID < 1 || _accountID == null) block and change your code as follows:
if (l != null)
{
_accountID = (int)l.AccountIDFK;
_securityLevelID = (int)l.SecurityLevelIDFK;
}
else
{
// logic when l is null
lbl_LoginStatus.Text = "Invalid";
}
Alternatively you can also use C# Ternary Operator to check if l is null
_accountID = l != null ? (int)l.AccountIDFK : 0;
_securityLevelID = l != null ? (int)l.SecurityLevelIDFK : 0;
if (_accountID < 1)
{
lbl_LoginStatus.Text = "Invalid";
}
You should check for null value in 'l' before using it.
if(l!=null)
{
_accountID = (int)l.AccountIDFK;
_securityLevelID = (int)l.SecurityLevelIDFK;
}
else
{
lbl_LoginStatus.Text = "Invalid";
}
Linq FirstOrDefault returns null if there is no item in the list matching your query.So if you get null in your code means that user login is invalid.
public int _accountID;
public int _securityLevelID;
public void GetLoginInfo(string EmailAddress, string Password)
{
LoginItem l = null;
{
try
{
using (RootsDataContext RDC = new RootsDataContext())
l = (from a in RDC.DBLogIns
where a.EmailAddress == EmailAddress
&& a.Password == Password
&& a.IsActive == 1
select new LoginItem
{
AccountIDFK = a.AccountIDFK,
SecurityLevelIDFK = a.SecurtityLevelIDFK,
}).FirstOrDefault();
if(l==null || _accountID < 1 || _accountID == null)
{
lbl_LoginStatus.Text = "Invalid";
Response.Redirect("~/InvalidCredentials.aspx"); // redirect to invalid login page.
}
else
{
_accountID = (int)l.AccountIDFK;
_securityLevelID = (int)l.SecurityLevelIDFK;
}
}
catch (Exception ex)
{
string error = ex.Message;
}
if (_accountID > 0)
{
if (_accountID == 1 && _securityLevelID == 1) // [Quentin]
{
Response.Redirect("~/AccountsMaster.aspx");
}
if (_accountID > 1 && _securityLevelID == 2) // [Companies]
{
Response.Redirect("~/CompanyMaster.aspx");
}
if (_accountID > 1 && _securityLevelID == 3) // [Branch]
{
Response.Redirect("~/BranchMaster.Aspx");
}
if (_accountID > 1 && _securityLevelID == 4) // [Clients]
{
Response.Redirect("~/Home.aspx");
}
}
}
}
You should have a check for LoginItem's default value null, and if its null (in case of invalid credentials) then do whatever you want.