I have this function where it opens a .TXT file with some products and insert line by line on the sqlitedb. The process is working fine, the problem is. This file contains 2000+ lines, because of that, the process is taking several hours to finish. I wonder if there is a way to make the process a little bit faster.
here is the function:
private void carrega_produtos()
{
var assembly = typeof(sincroniza_page).GetTypeInfo().Assembly;
foreach (var res in assembly.GetManifestResourceNames())
{
if (res.Contains("produtos.txt"))
{
Stream stream = assembly.GetManifestResourceStream(res);
var st = res.Count();
using (var reader = new StreamReader(stream))
{
string linha;
acesso_banco_produtos banco = new acesso_banco_produtos();
while ((linha = reader.ReadLine()) != null)
{
List<string> lista = linha.Split(new char[] { 'ยง' }).ToList();
var cod = int.Parse(lista.ElementAt(0));
var nome_prod = lista.ElementAt(1);
var cod_grupo = lista.ElementAt(2);
var nm_grupo = lista.ElementAt(3);
var ind_ativo = lista.ElementAt(4);
var val_custo_unit = lista.ElementAt(5);
var val_custo = lista.ElementAt(6);
var perc_imposto = lista.ElementAt(7);
var unidade_med = lista.ElementAt(8);
var qtd_mes_1 = lista.ElementAt(9);
var qtd_mes_2 = lista.ElementAt(10);
var qtd_mes_3 = lista.ElementAt(11);
var qtd_mes_6 = lista.ElementAt(12);
var qtd_mes_12 = lista.ElementAt(13);
var data = lista.ElementAt(14);
var bd = new banco_produtos()
{
cod_produto = cod,
nm_produto = nome_prod,
cod_grupo = cod_grupo,
nm_grupo = nm_grupo,
ind_ativo = ind_ativo,
val_custo_unitario = Double.Parse(val_custo_unit),
val_lista_preco = val_custo,
perc_impostos = perc_imposto,
unidade_medida = unidade_med,
qtde_vendida_mes_1 = qtd_mes_1,
qtde_vendida_mes_2 = qtd_mes_2,
qtde_vendida_mes_3 = qtd_mes_3,
qtde_vendida_mes_6 = qtd_mes_6,
qtde_vendida_mes_12 = qtd_mes_12
};
//here i look in the DB if already exists the new product
var procura = banco.get_produto(cod);
if (procura == null)
{
// here is inserted to the db
banco.inserir_produto(bd);
}
}
valor += 1;
}
}
}
}
I'm not sure what is inside your method which inserts data into db but the most common issue with SQLite and massive inserts is the fact that SQLite by default wraps every insert with transaction which creates significant overhead. A good practice for such cases is to make signle transaction for all insterts which should singificantly improve the perfomance, see the example.
I did what #Dmytro said, I used the method "insertORIgnore". It improved a lot using that method.thank you for the help.
Related
I have a list of 'Sites' that are stored in my database. The list is VERY big and contains around 50,000+ records.
I am trying to loop through each record and update it. This takes ages, is there a better more efficient way of doing this?
using (IRISInSiteLiveEntities DB = new IRISInSiteLiveEntities())
{
var allsites = DB.Sites.ToList();
foreach( var sitedata in allsites)
{
var siterecord = DB.Sites.Find(sitedata.Id);
siterecord.CabinOOB = "Test";
siterecord.TowerOOB = "Test";
siterecord.ManagedOOB = "Test";
siterecord.IssueDescription = "Test";
siterecord.TargetResolutionDate = "Test";
DB.Entry(siterecord).State = EntityState.Modified;
}
DB.SaveChanges();
}
I have cut the stuff out of the code to get to the point. The proper function code I am using basically pulls a list out from Excel, then matches the records in the sites list and updates each record that matches accordingly. The DB.Find is slowing the loop down dramatically.
[HttpPost]
public ActionResult UploadUpdateOOBList()
{
CheckPermissions("UpdateOOBList");
string[] typesallowed = new string[] { ".xls", ".xlsx" };
HttpPostedFileBase file = Request.Files[0];
var fname = file.FileName;
if (!typesallowed.Any(fname.Contains))
{
return Json("NotAllowed");
}
file.SaveAs(Server.MapPath("~/Uploads/OOB List/") + fname);
//Create empty OOB data list
List<OOBList.OOBDetails> oob_data = new List<OOBList.OOBDetails>();
//Using ClosedXML rather than Interop Excel....
//Interop Excel: 30 seconds for 750 rows
//ClosedXML: 3 seconds for 750 rows
string fileName = Server.MapPath("~/Uploads/OOB List/") + fname;
using (var excelWorkbook = new XLWorkbook(fileName))
{
var nonEmptyDataRows = excelWorkbook.Worksheet(2).RowsUsed();
foreach (var dataRow in nonEmptyDataRows)
{
//for row number check
if (dataRow.RowNumber() >= 4 )
{
string siteno = dataRow.Cell(1).GetValue<string>();
string sitename = dataRow.Cell(2).GetValue<string>();
string description = dataRow.Cell(4).GetValue<string>();
string cabinoob = dataRow.Cell(5).GetValue<string>();
string toweroob = dataRow.Cell(6).GetValue<string>();
string manageoob = dataRow.Cell(7).GetValue<string>();
string resolutiondate = dataRow.Cell(8).GetValue<string>();
string resolutiondate_converted = resolutiondate.Substring(resolutiondate.Length - 9);
oob_data.Add(new OOBList.OOBDetails
{
SiteNo = siteno,
SiteName = sitename,
Description = description,
CabinOOB = cabinoob,
TowerOOB = toweroob,
ManageOOB = manageoob,
TargetResolutionDate = resolutiondate_converted
});
}
}
}
//Now delete file.
System.IO.File.Delete(Server.MapPath("~/Uploads/OOB List/") + fname);
Debug.Write("DOWNLOADING LIST ETC....\n");
using (IRISInSiteLiveEntities DB = new IRISInSiteLiveEntities())
{
var allsites = DB.Sites.ToList();
//Loop through sites and the OOB list and if they match then tell us
foreach( var oobdata in oob_data)
{
foreach( var sitedata in allsites)
{
var indexof = sitedata.SiteName.IndexOf(' ');
if( indexof > 0 )
{
var OOBNo = oobdata.SiteNo;
var OOBName = oobdata.SiteName;
var SiteNo = sitedata.SiteName;
var split = SiteNo.Substring(0, indexof);
if (OOBNo == split && SiteNo.Contains(OOBName) )
{
var siterecord = DB.Sites.Find(sitedata.Id);
siterecord.CabinOOB = oobdata.CabinOOB;
siterecord.TowerOOB = oobdata.TowerOOB;
siterecord.ManagedOOB = oobdata.ManageOOB;
siterecord.IssueDescription = oobdata.Description;
siterecord.TargetResolutionDate = oobdata.TargetResolutionDate;
DB.Entry(siterecord).State = EntityState.Modified;
Debug.Write("Updated Site ID/Name Record: " + sitedata.Id + "/" + sitedata.SiteName);
}
}
}
}
DB.SaveChanges();
}
var nowdate = DateTime.Now.ToString("dd/MM/yyyy");
System.IO.File.WriteAllText(Server.MapPath("~/Uploads/OOB List/lastupdated.txt"),nowdate);
return Json("Success");
}
Looks like you are using Entity Framework (6 or Core). In either case both
var siterecord = DB.Sites.Find(sitedata.Id);
and
DB.Entry(siterecord).State = EntityState.Modified;
are redundant, because the siteData variable is coming from
var allsites = DB.Sites.ToList();
This not only loads the whole Site table in memory, but also EF change tracker keeps reference to every object from that list. You can easily verify that with
var siterecord = DB.Sites.Find(sitedata.Id);
Debug.Assert(siterecord == sitedata);
The Find (when the data is already in memory) and Entry methods themselves are fast. But the problem is that they by default trigger automatic DetectChanges, which leads to quadratic time complexity - in simple words, very slow.
With that being said, simply remove them:
if (OOBNo == split && SiteNo.Contains(OOBName))
{
sitedata.CabinOOB = oobdata.CabinOOB;
sitedata.TowerOOB = oobdata.TowerOOB;
sitedata.ManagedOOB = oobdata.ManageOOB;
sitedata.IssueDescription = oobdata.Description;
sitedata.TargetResolutionDate = oobdata.TargetResolutionDate;
Debug.Write("Updated Site ID/Name Record: " + sitedata.Id + "/" + sitedata.SiteName);
}
This way EF will detect changes just once (before SaveChanges) and also will update only the modified record fields.
I have followed Ivan Stoev's suggestion and have changed the code by removing the DB.Find and the EntitySate Modified - It now takes about a minute and a half compared to 15 minutes beforehand. Very suprising as I didn't know that you dont actually require that to update the records. Clever. The code is now:
using (IRISInSiteLiveEntities DB = new IRISInSiteLiveEntities())
{
var allsites = DB.Sites.ToList();
Debug.Write("Starting Site Update loop...");
//Loop through sites and the OOB list and if they match then tell us
//750 records takes around 15-20 minutes.
foreach( var oobdata in oob_data)
{
foreach( var sitedata in allsites)
{
var indexof = sitedata.SiteName.IndexOf(' ');
if( indexof > 0 )
{
var OOBNo = oobdata.SiteNo;
var OOBName = oobdata.SiteName;
var SiteNo = sitedata.SiteName;
var split = SiteNo.Substring(0, indexof);
if (OOBNo == split && SiteNo.Contains(OOBName) )
{
sitedata.CabinOOB = oobdata.CabinOOB;
sitedata.TowerOOB = oobdata.TowerOOB;
sitedata.ManagedOOB = oobdata.ManageOOB;
sitedata.IssueDescription = oobdata.Description;
sitedata.TargetResolutionDate = oobdata.TargetResolutionDate;
Debug.Write("Thank you, next: " + sitedata.Id + "\n");
}
}
}
}
DB.SaveChanges();
}
So first of all you should turn your HTTPPost in an async function
more info https://learn.microsoft.com/en-us/dotnet/csharp/programming-guide/concepts/async/
What you then should do is create the tasks and add them to a list. Then wait for them to complete (if you want/need to) by calling Task.WaitAll()
https://learn.microsoft.com/en-us/dotnet/api/system.threading.tasks.task.waitall?view=netframework-4.7.2
This will allow your code to run in parallel on multiple threads optimizing performance quite a bit already.
You can also use linq to for example reduce the size of allsites beforehand by doing something that will roughly look like this
var sitedataWithCorrectNames = allsites.Where(x => x //evaluate your condition here)
https://learn.microsoft.com/en-us/dotnet/framework/data/adonet/ef/language-reference/supported-and-unsupported-linq-methods-linq-to-entities
and then start you foreach (var oobdata) with the now foreach(sitedate in sitedataWithCorrectNames)
Same goes for SiteNo.Contains(OOBName)
https://learn.microsoft.com/en-us/dotnet/csharp/programming-guide/concepts/linq/getting-started-with-linq
P.S. Most db sdk's also provide asynchornous functions so use those aswell.
P.P.S. I didn't have an IDE so I eyeballed the code but the links should provide you with plenty of samples. Reply if you need more help.
So, I am trying to send some money over using NBitcoin, there is a step where i am failing and that is creating de bitcoin secret to sign the transaction, I have the address, and the ExtPrivKey but i haven't gotten any luck signing it, any recommendation, this is my code below.
var priv = mbwallet.SelectedWallet.PrivateKeys[0].ToWif();
//var ool = new BitcoinSecret(base58, App.Network);
var privkey = mbwallet.SelectedWallet.PrivateKeys[0].PrivateKey.GetBitcoinSecret(App.Network).ToWif();
var key = Key.Parse(privkey, App.Network);
var keysT = key.GetWif(App.Network);
//var myaddress = mbwallet.SelectedWallet.PrivateKeys[0].PrivateKey.PubKey.GetAddress(App.Network);
var myaddress = mbwallet.SelectedWallet.CurrentAddress;
string address = Address.Text;
var destination = BitcoinAddress.Create(address, App.Network);
decimal value = Convert.ToDecimal(Value.Text);
var coins2 = GetCoins(value);
TransactionBuilder txBuilder = new TransactionBuilder();
var tx = txBuilder.AddCoins(coins2)
.AddKeys(keysT)
.SetChange(myaddress)
.Send(destination, new Money(value, MoneyUnit.BTC))
.SendFees("0.0002");
//.BuildTransaction(true);
var tx2 = txBuilder.BuildTransaction(true);
//Console.WriteLine(txBuilder.Verify(tx));
var hello = tx2.ToHex();
var txRepo = new NoSqlTransactionRepository();
//txRepo.Put(tx.GetHash(), tx);
//Assert(txBuilder.Verify(tx)); //check fully signed
List<ICoin> GetCoins(decimal sendAmount)
{
//var mbwallet = (root.DataContext as MainWindowViewModel);
var amountMoney = new Money(sendAmount, MoneyUnit.BTC);
var client = new QBitNinjaClient(App.Network);
var txInAmount = Money.Zero;
var coins1 = new List<ICoin>();
foreach (var balance in client.GetBalance(mbwallet.SelectedWallet.CurrentAddress,//MBWallet.Wallet.Address,
true).Result.Operations)
{
var transactionId = balance.TransactionId;
var transactionResponse =
client.GetTransaction(transactionId).Result;
var receivedCoins = transactionResponse.ReceivedCoins;
foreach (Coin coin in receivedCoins)
{
if (coin.TxOut.ScriptPubKey ==
mbwallet.SelectedWallet.CurrentAddress.ScriptPubKey)//MBWallet.Wallet.BitcoinPrivateKey.ScriptPubKey) // this may not be necessary
{
coins1.Add(coin);
txInAmount += (coin.Amount as Money);
}
}
}
return coins1;
}
For what I see in the code you already add the private key to the builder so basically you only need to sign , something like this
Transaction signed = txBuilder.SignTransaction(tx2);
I've put together a CSV importer which I assume works, though I get this error, how do I allow this column to be null so when it adds it to the table it automatically sets the ID? I've tried:
csv.Configuration.WillThrowOnMissingFields = false;
but it doesn't recognise it, this is the error I get when attempting to upload:
CsvHelper.ValidationException: 'Header matching ['ID'] names at index 0 was not found. If you are expecting some headers to be missing and want to ignore this validation, set the configuration HeaderValidated to null. You can also change the functionality to do something else, like logging the issue.'
[HttpPost]
[ActionName("CreateBulk")]
public ActionResult CreateBulkUpload()
{
object db;
var file = Request.Files["attachmentcsv"];
using (var csv = new CsvReader(new StreamReader(file.InputStream), true))
{
var records = csv.GetRecords<Client>().ToList();
foreach (var item in records)
{
var strip = item.homePage.Replace("https://www.", "").Replace("http://www.", "")
.Replace("https://", "").Replace("http://", "").Replace("www.", "");
string[] URLtests =
{"https://www." + strip, "http://www." + strip, "https://" + strip, "http://" + strip};
string[] Metric = MajesticFunctions.MajesticChecker(URLtests);
var userId = User.Identity.GetHashCode();
var UserTableID = 1;
var newclient = new Client
{
clientN = item.clientN,
homePage = Metric[0],
clientEmail = item.clientEmail,
monthlyQuota = item.monthlyQuota,
TrustFlow = Int32.Parse(Metric[1]),
CitationFlow = Int32.Parse(Metric[2]),
RI = Int32.Parse(Metric[3]),
MJTopicsID = item.MJTopicsID,
UserTableID = UserTableID
};
ViewBag.newdomain = newclient;
return RedirectToAction("Index");
}
}
return RedirectToAction("Index");
}
Did you try out the suggestion mentioned in the error message?
like this?
csv.configuration.HeaderValidated = null;
The developer made some breaking changes this year, so the accepted answer will no longer work.
Instead, you have to create a configuration object in advance and inject it in the constructor:
var config = new CsvConfiguration(CultureInfo.InvariantCulture)
{
HeaderValidated = null
};
using (var reader = new StreamReader(file))
using (var csv = new CsvReader(reader, config))
Make sure to include both these lines:
csv.Configuration.HeaderValidated = null;
csv.Configuration.MissingFieldFound = null;
I am not able to add or update milestones field for the Features in the Rally. If anyone having the code available using C# to update the same, please share with me. I am searching and doing from last one week with no luck.
When I am trying to add/Update milestones in the Features. I am getting the error as "Could not read: Could not read referenced object null". My code is as follows:-
public DynamicJsonObject UpdateFeaturesbyName(string fea, string bFun)
{
//getting list of Feature.
Request feat = new Request("PortfolioItem/Feature");
feat.Query = new Query("Name", Query.Operator.Equals, fea);
QueryResult TCSResults = restApi.Query(feat);
foreach (var res in TCSResults.Results)
{
var steps = res["Milestones"];
Request tsteps = new Request(steps);
QueryResult tstepsResults = restApi.Query(tsteps);
foreach (var item in tstepsResults.Results)
{
}
if (res.Name == fea)
{
var targetFeature = TCSResults.Results.FirstOrDefault();
DynamicJsonObject toUpdate = new DynamicJsonObject();
//toUpdate["Milestones"] = "";
// CreateResult createResult = restApi.Create(steps._ref, toUpdate);
// String contentRef = steps._ref;
//String contentRef = createResult._ref;
string[] value = null;
string AccCri = string.Empty;
if (!string.IsNullOrWhiteSpace(bFun))
{
value = bFun.Split(new string[] { "\r\n", "\n" }, StringSplitOptions.None);
foreach (string item in value)
{
//if (string.IsNullOrWhiteSpace(AccCri))
// AccCri = item;
//else
// AccCri = AccCri + "<br/>" + item;
if (!string.IsNullOrWhiteSpace(item))
{
//Query for Milestone.
Request ms = new Request("Milestone");
ms.Fetch = new List<string>() { "Name", "ObjectID" };
ms.Query = new Query("Name", Query.Operator.Equals, item);
QueryResult msResults = restApi.Query(ms);
var targetMLResult = msResults.Results.FirstOrDefault();
long MLOID = targetMLResult["ObjectID"];
DynamicJsonObject tarML = restApi.GetByReference("Milestone", MLOID, "Name", "_ref", "DisplayColor");
DynamicJsonObject targetML = new DynamicJsonObject();
targetML["Name"] = tarML["Name"];
//targetML["_ref"] = tarML["_ref"];
targetML["_ref"] = "/milestone/" + Convert.ToString(MLOID);
targetML["DisplayColor"] = tarML["DisplayColor"];
// Grab collection of existing Milestones.
var existingMilestones = targetFeature["Milestones"];
long targetOID = targetFeature["ObjectID"];
// Milestones collection on object is expected to be a System.Collections.ArrayList.
var targetMLArray = existingMilestones;
var tagList2 = targetMLArray["_tagsNameArray"];
tagList2.Add(targetML);//
//targetMLArray.Add(targetML);
targetMLArray["_tagsNameArray"] = tagList2;
toUpdate["Milestones"] = targetMLArray;
OperationResult updateResult = restApi.Update(res._ref, toUpdate);
bool resp = updateResult.Success;
}
}
}
//toUpdate["c_AcceptanceCriteria"] = AccCri;
//OperationResult updateResult = restApi.Update(res._ref, toUpdate);
}
}
var features = TCSResults.Results.Where(p => p.Name == fea).FirstOrDefault();
var featuresref = features._ref;
return features;
}
Now that v3.1.1 of the toolkit has been released you can use the AddToCollection method to do this.
Otherwise, you can still always just update the full collection. The value should be an arraylist of objects with _ref properties.
Check out this example (which adds tasks to defects, but should be very similar to what you're doing): https://github.com/RallyCommunity/rally-dot-net-rest-apps/blob/master/UpdateTaskCollectionOnDefect/addTaskOnDefect.cs
The application I am building allows a user to upload a .csv file, which will ultimately fill in fields of an existing SQL table where the Ids match. First, I am using LinqToCsv and a foreach loop to import the .csv into a temporary table. Then I have another foreach loop that loops the fields from the temporary table into an existing table where the Ids match. The only way I have gotten this to work consistently and successfully is nesting the second foreach loop within the first:
[HttpPost]
public ActionResult UploadValidationTable(HttpPostedFileBase csvFile)
{
var inputFileDescription = new CsvFileDescription
{
SeparatorChar = ',',
FirstLineHasColumnNames = true
};
var cc = new CsvContext();
var filePath = uploadFile(csvFile.InputStream);
var model = cc.Read<Credit>(filePath, inputFileDescription);
try
{
var entity = new TestEntities();
foreach (var item in model)
{
var tc = new TemporaryCsvUpload
{
Id = item.Id,
CreditInvoiceAmount = item.CreditInvoiceAmount,
CreditInvoiceDate = item.CreditInvoiceDate,
CreditInvoiceNumber = item.CreditInvoiceNumber,
CreditDeniedDate = item.CreditDeniedDate,
CreditDeniedReasonId = item.CreditDeniedReasonId,
CreditDeniedNotes = item.CreditDeniedNotes
};
entity.TemporaryCsvUploads.Add(tc);
var idMatches = entity.Authorizations.ToList().Where(x => x.Id == tc.Id);
foreach (var number in idMatches)
{
number.CreditInvoiceDate = tc.CreditInvoiceDate;
number.CreditInvoiceNumber = tc.CreditInvoiceNumber;
number.CreditInvoiceAmount = tc.CreditInvoiceAmount;
number.CreditDeniedDate = tc.CreditDeniedDate;
number.CreditDeniedReasonId = tc.CreditDeniedReasonId;
number.CreditDeniedNotes = tc.CreditDeniedNotes;
}
}
entity.SaveChanges();
entity.Database.ExecuteSqlCommand("TRUNCATE TABLE TemporaryCsvUpload");
TempData["Success"] = "Updated Successfully";
}
catch (LINQtoCSVException)
{
TempData["Error"] = "Upload Error: Ensure you have the correct header fields and that the file is of .csv format.";
}
return View("Upload");
}
The issue is speed. It takes about 1 minute and 49 seconds to search through an SQL table of 7000 entries, match the ids, and fill in the fields.
So, I looked at this and thought that the second loop really didn't need to be nested. I switched up the code like so:
[HttpPost]
public ActionResult UploadValidationTable(HttpPostedFileBase csvFile)
{
var inputFileDescription = new CsvFileDescription
{
SeparatorChar = ',',
FirstLineHasColumnNames = true
};
var cc = new CsvContext();
var filePath = uploadFile(csvFile.InputStream);
var model = cc.Read<Credit>(filePath, inputFileDescription);
try
{
var entity = new TestEntities();
var tc = new TemporaryCsvUpload();
foreach (var item in model)
{
tc.Id = item.Id;
tc.CreditInvoiceAmount = item.CreditInvoiceAmount;
tc.CreditInvoiceDate = item.CreditInvoiceDate;
tc.CreditInvoiceNumber = item.CreditInvoiceNumber;
tc.CreditDeniedDate = item.CreditDeniedDate;
tc.CreditDeniedReasonId = item.CreditDeniedReasonId;
tc.CreditDeniedNotes = item.CreditDeniedNotes;
entity.TemporaryCsvUploads.Add(tc);
}
var idMatches = entity.Authorizations.ToList().Where(x => x.Id == tc.Id);
foreach (var number in idMatches)
{
number.CreditInvoiceDate = tc.CreditInvoiceDate;
number.CreditInvoiceNumber = tc.CreditInvoiceNumber;
number.CreditInvoiceAmount = tc.CreditInvoiceAmount;
number.CreditDeniedDate = tc.CreditDeniedDate;
number.CreditDeniedReasonId = tc.CreditDeniedReasonId;
number.CreditDeniedNotes = tc.CreditDeniedNotes;
}
entity.SaveChanges();
entity.Database.ExecuteSqlCommand("TRUNCATE TABLE TemporaryCsvUpload");
TempData["Success"] = "Updated Successfully";
}
catch (LINQtoCSVException)
{
TempData["Error"] = "Upload Error: Ensure you have the correct header fields and that the file is of .csv format.";
}
return View("Upload");
}
This time around, it only took 19 seconds to complete. A vast improvement on the first. But when I checked the database, only one row of the 7 that should match was filled in. Can anybody spot a reason why the second code block would not be filling in all the rows it should be? Or a better way to optimize the first block? Thanks!