The following code is taking ~2 minutes to delete 30k records which I am sure is too long. Most of the similar questions I have seen on here have been solved by using a SQLiteTransaction object, but I am already doing that.
private void removeProxiesButton_Click(object sender, EventArgs e)
{
using (var conn = new SQLiteConnection(Properties.Settings.Default.dbConnectionString))
{
conn.Open();
using (var trans = conn.BeginTransaction())
{
using (var cmd = new SQLiteCommand("DELETE FROM Proxy WHERE IP=#ip AND Port=#port", conn, trans))
{
foreach (DataGridViewRow row in proxiesDataGridView.SelectedRows)
{
var proxy = proxies[row.Index];
cmd.Parameters.AddWithValue("#ip", proxy.IP);
cmd.Parameters.AddWithValue("#port", proxy.Port);
cmd.ExecuteNonQuery();
proxies.Remove(proxy);
}
}
trans.Commit();
}
}
}
And here is the CREATE statement for the Proxy table.
CREATE TABLE "Proxy"
(
`ProxyID` INTEGER PRIMARY KEY AUTOINCREMENT UNIQUE,
`Status` TEXT,
`IP` TEXT,
`Port` INTEGER,
`Country` TEXT,
`Speed` INTEGER,
`DateAdded` TEXT
)
Building one long SQLite statement with a StringBuilder and only executing the SQLiteCommand object once sped things up significantly.
private void removeProxiesButton_Click(object sender, EventArgs e)
{
using (var conn = new SQLiteConnection(Properties.Settings.Default.dbConnectionString))
{
conn.Open();
var sb = new StringBuilder();
sb.Append("DELETE FROM Proxy WHERE ProxyID IN (");
foreach (DataGridViewRow row in proxiesDataGridView.SelectedRows)
{
var proxy = proxies[row.Index];
sb.Append(proxy.ProxyID + ",");
}
sb[sb.Length - 1] = ')';
using (var cmd = new SQLiteCommand(sb.ToString(), conn))
{
cmd.ExecuteNonQuery();
}
}
}
Related
I have a class named ImageData who contains a list of Tags
I get the database locked error only if an image has more than 1 tag and I can't find out why
0 tag and 1 tag is always fine, with 1 image or a 100.
As soon as 1 image has 2 tags, I get the error
I make sure of disposing of everything with the using statement
here is the method
public static int addImages(List<ImageData> images)
{
int rows = 0;
using (SQLiteConnection con = new SQLiteConnection(ConnectionString()))
{
con.Open();
foreach (ImageData img in images)
{
using (SQLiteCommand cmd = new SQLiteCommand($"INSERT INTO images(Hash, Extension, Name) VALUES(#IHash,#IExtension, #IName)", con))
{
cmd.Parameters.AddWithValue("#IHash", img.Hash);
cmd.Parameters.AddWithValue("#IExtension", img.Extension);
cmd.Parameters.AddWithValue("#IName", img.Name);
rows += cmd.ExecuteNonQuery();
}
foreach (Tag tag in img.Tags)
{
using (SQLiteCommand cmd = new SQLiteCommand($"INSERT INTO ImagesTags(ImageHash, TagName) VALUES(#IHash,#IName)", con))
{
cmd.Parameters.AddWithValue("#IHash", img.Hash);
cmd.Parameters.AddWithValue("#IName", tag.Name);
cmd.ExecuteNonQuery();
}
}
}
con.Close();
}
return rows;
}
here are my tables Creation
string[] createTable =
{
"CREATE TABLE images(Hash TEXT PRIMARY KEY, Extension TEXT, Name TEXT)",
"CREATE TABLE tags(NAME TEXT PRIMARY KEY NOT NULL,DESCRIPTION TEXT,COLLECTIONNAME TEXT)",
"CREATE TABLE ImagesTags(ImageHash TEXT,TagName TEXT,Primary KEY (ImageHash, TagName),FOREIGN KEY (ImageHash) REFERENCES images(Hash),FOREIGN KEY (TagName) REFERENCES tags(Name))"
};
There are multiple cases where I insert data in a foreach loop and this is the only place where I get this error.
After googling quite a lot I learned about cleaner ways of working with sqlite.
The probleme was fixed by using using statement for every connections, commands, transactions and readers.
something somewhere must have been incorrectly disposed
here is the same function as before but with better coding
I also added transactions to make only 1 call to the database and allow a rollback if something went wrong
public static int addImages(List<ImageData> images)
{
int rows = 0;
using (var con = new SQLiteConnection(ConnectionString()))
{
con.Open();
using (var tra = con.BeginTransaction())
{
try
{
foreach (ImageData img in images)
{
SQLiteParameter p1 = new SQLiteParameter("#IHash", System.Data.DbType.String);
SQLiteParameter p2 = new SQLiteParameter("#IExtension", System.Data.DbType.String);
SQLiteParameter p3 = new SQLiteParameter("#IName", System.Data.DbType.String);
using (SQLiteCommand cmd = new SQLiteCommand($"INSERT INTO images(Hash, Extension, Name) VALUES(#IHash,#IExtension, #IName)", tra.Connection))
{
cmd.Parameters.Add(p1);
cmd.Parameters.Add(p2);
cmd.Parameters.Add(p3);
p1.Value = img.Hash;
p2.Value = img.Extension;
p3.Value = img.Name;
cmd.ExecuteNonQuery();
}
foreach (Tag tag in img.Tags)
{
using (SQLiteCommand cmd = new SQLiteCommand($"INSERT INTO ImagesTags(ImageHash, TagName) VALUES(#IHash,#IName)", tra.Connection))
{
cmd.Parameters.AddWithValue("#IHash", img.Hash);
cmd.Parameters.AddWithValue("#IName", tag.Name);
cmd.ExecuteNonQuery();
}
}
}
tra.Commit();
}
catch(Exception ex)
{
tra.Rollback();
throw;
}
}
}
return rows;
}
I've got a problem with some queries from c# to SQL. I need to have a query executeReader and inside of it a if else that allow me to choose between two inserts queries. I'm calling a little external program(with the URL collected into the db) which allows me to choose between 1 and 2, if the 1 is chosen(pass) else(fail). I can't do that because the debug is giving me:
'A Command is yet associated with a DataReader opened, which needs to be closed.'
I don't know what to try anymore.
private void btnSTART_Click(object sender, RoutedEventArgs e)
{
sqliteCon.Open();
if (sqliteCon.State == System.Data.ConnectionState.Open)
{
string path = null;//estrazione1
SqlCommand cmd = new SqlCommand("select nomeI FROM tabL where selection=1", sqliteCon);
SqlDataReader nomeIRdr = null;//estrazione2
//qui
var scriptsToRun = new List<string>();
using (nomeIRdr = cmd.ExecuteReader())
{
while (nomeIRdr.Read())//estrazione4
{
path = nomeIRdr["nomeI"].ToString();//estrazione5
Process MyProc = Process.Start(path);//permette la run del path contenuto nel db
MyProc.WaitForExit();
var exitCode = MyProc.ExitCode;
if (exitCode == 1)
{
scriptsToRun.Add("insert into tabL resItem values 'PASS'");
}
else
{
scriptsToRun.Add("insert into tabL resItem values 'FAIL'");
}
sqliteCon.Close();
}
}
foreach (var script in scriptsToRun)
{
SqlCommand cmd1 = new SqlCommand(script, sqliteCon);
cmd1.ExecuteNonQuery();
}
}
}
Do not share single connection and cram everything into a single routine. Please, keep your code simple.
Create (and Dispose) Connection whenever you query RDBMS
Extract methods
Code:
Process execution and return execution results collection:
// Item1 - path
// Item2 - true in succeed
private List<Tuple<string, bool>> ExecuteResults() {
List<Tuple<string, bool>> result = new List<Tuple<string, bool>>();
using (var con = new SqlConnection(ConnectionStringHere)) {
con.Open();
string sql =
#"select nomeItem
from tabList
where selection = 1";
using (SqlCommand cmd = new SqlCommand(sql, con)) {
using (var reader = cmd.ExecuteReader()) {
while (reader.Read()) {
string path = Convert.ToString(reader[0]);
using (Process process = Process.Start(path)) {
process.WaitForExit();
result.Add(Tuple.Create(path, process.ExitCode == 1));
}
}
}
}
}
return result;
}
Saving results in RDBMS
private void ApplyExecuteResults(IEnumerable<Tuple<string, bool>> results) {
using (var con = new SqlConnection(ConnectionStringHere)) {
con.Open();
string sql =
#"update tabList
set resItem = #prm_resItem
where nomeItem = #prm_nomeItem";
using (SqlCommand cmd = new SqlCommand(sql, con)) {
cmd.Parameters.Add("#prm_nomeItem", SqlDbType.VarChar);
cmd.Parameters.Add("#prm_resItem", SqlDbType.VarChar);
foreach (var item in results) {
cmd.Parameters[0].Value = item.Item1;
cmd.Parameters[1].Value = item.Item2 ? "PASS" : "FAIL";
cmd.ExecuteNonQuery();
}
}
}
}
Finally, combine both routines:
private void btnSTART_Click(object sender, RoutedEventArgs e) {
ApplyExecuteResults(ExecuteResults());
}
I am trying to doing this:
Read a row from an SQLite db (in GetRuleByID() method)
Update the same row that I just read during (1) (See UpdateStatusForRuleID() method)
However my problem is that SQLite locks the database after the SELECT in GetRuleByID() so that update in UpdateStatusForRuleID() is only successful when called the first time.
I have tried enabling Write-Ahead-Logging in SQLite as well as PRAGMA read_uncommitted=1 in order to avoid SQLite locking the database for the SELECT, but this does not appear to work.
This should be simple but I have so far spent a complete night trying to solve this... Please help !
private static MicroRuleEngine.Rule GetRuleByID(int ruleID, SQLiteConnection connection, out Dictionary<string, string> dict)
{
dict = new Dictionary<string, string>();
string sql = String.Format("select * from rules WHERE ID = {0} ", ruleID.ToString());
SQLiteCommand command = new SQLiteCommand(sql, connection);
SQLiteDataReader reader = command.ExecuteReader();
if (reader.HasRows)
{
reader.Read();
// Convert row into a dictionary
for (int lp = 0; lp < reader.FieldCount; lp++)
{
dict.Add(reader.GetName(lp), reader.GetValue(lp) as string);
}
string json = dict["fulljson"];
MicroRuleEngine.Rule r = Newtonsoft.Json.JsonConvert.DeserializeObject<MicroRuleEngine.Rule>(json);
//command.Dispose();
return r;
}
}
internal static void UpdateStatusForRuleID(SQLConnectionManager DBMANAGER, int ruleID, bool status)
{
Dictionary<string, string> dict = null;
string dbVal = (status) ? "1" : "0";
MicroRuleEngine.Rule r = null;
string newJSON = null;
using (SQLiteConnection connection = DBMANAGER.CreateConnection())
{
r = GetRuleByID(ruleID, connection, out dict);
r.Active = (status);
newJSON = Newtonsoft.Json.JsonConvert.SerializeObject(r);
Thread.Sleep(1000);
string sql = "UPDATE rules SET active = #a, fulljson=#j WHERE ID = #i";
using (var command = new SQLiteCommand(sql, connection))
{
command.Parameters.Add(new SQLiteParameter("#a", dbVal));
command.Parameters.Add(new SQLiteParameter("#i", ruleID));
command.Parameters.Add(new SQLiteParameter("#j", newJSON));
command.ExecuteNonQuery(); // Database is locked here ???
}
connection.Close();
}
}
"Database is locked" means that some other connection (in the same or some other process) still has an active transaction.
You don't need multiple connections (unless you are using multiple threads); just use a single connection object for all database accesses.
Ensure that all command, reader, and transaction objects (and connections, if you decide to use temporary ones) are properly cleaned up, by using using:
using (var command = new SQLiteCommand(sql, connection))
using (var reader = command.ExecuteReader())
{
if (reader.HasRows)
...
}
Apparently, the code below works. I basically dropped the GetRuleByID() method (but then I had to re-write 4 other methods)
Thanks to all who provided input.
internal static void UpdateStatusForRuleID(SQLConnectionManager DBMANAGER, int ruleID, bool status)
{
string dbVal = (status) ? "1" : "0";
MicroRuleEngine.Rule r = null;
string newJSON = null;
using (SQLiteConnection conn = DBMANAGER.CreateConnection())
{
string sql = String.Format("select * from rules WHERE ID = {0} ", ruleID.ToString());
using (var command = new SQLiteCommand(sql, conn))
using (var reader = command.ExecuteReader())
{
if (reader.HasRows)
{
reader.Read();
string json = reader["fulljson"].ToString();
r = Newtonsoft.Json.JsonConvert.DeserializeObject<MicroRuleEngine.Rule>(json);
r.Active = (status);
newJSON = Newtonsoft.Json.JsonConvert.SerializeObject(r);
string sql2 = "UPDATE rules SET active = #a, fulljson=#j WHERE ID = #i";
using (var command2 = new SQLiteCommand(sql2, conn))
{
command2.Parameters.Add(new SQLiteParameter("#a", dbVal));
command2.Parameters.Add(new SQLiteParameter("#i", ruleID));
command2.Parameters.Add(new SQLiteParameter("#j", newJSON));
command2.ExecuteNonQuery();
}
}
}
}
}
in my code the user can upload an excel document wish contains it's phone contact list.Me as a developer should read that excel file turn it into a dataTable and insert it into the database .
The Problem is that some clients have a huge amount of contacts like saying 5000 and more contacts and when i am trying to insert this amount of data into the database it's crashing and giving me a timeout exception.
What would be the best way to avoid this kind of exception and is their any code that can reduce the time of the insert statement so the user don't wait too long ?
the code
public SqlConnection connection = new SqlConnection(System.Configuration.ConfigurationManager.ConnectionStrings["ConnectionString"].ConnectionString);
public void Insert(string InsertQuery)
{
SqlDataAdapter adp = new SqlDataAdapter();
adp.InsertCommand = new SqlCommand(InsertQuery, connection);
if (connection.State == System.Data.ConnectionState.Closed)
{
connection.Open();
}
adp.InsertCommand.ExecuteNonQuery();
connection.Close();
}
protected void submit_Click(object sender, EventArgs e)
{
string UploadFolder = "Savedfiles/";
if (Upload.HasFile) {
string fileName = Upload.PostedFile.FileName;
string path=Server.MapPath(UploadFolder+fileName);
Upload.SaveAs(path);
Msg.Text = "successfully uploaded";
DataTable ValuesDt = new DataTable();
ValuesDt = ConvertExcelFileToDataTable(path);
Session["valuesdt"] = ValuesDt;
Excel_grd.DataSource = ValuesDt;
Excel_grd.DataBind();
}
}
protected void SendToServer_Click(object sender, EventArgs e)
{
DataTable Values = Session["valuesdt"] as DataTable ;
if(Values.Rows.Count>0)
{
DataTable dv = Values.DefaultView.ToTable(true, "Mobile1", "Mobile2", "Tel", "Category");
double Mobile1,Mobile2,Tel;string Category="";
for (int i = 0; i < Values.Rows.Count; i++)
{
Mobile1 =Values.Rows[i]["Mobile1"].ToString()==""?0: double.Parse(Values.Rows[i]["Mobile1"].ToString());
Mobile2 = Values.Rows[i]["Mobile2"].ToString() == "" ? 0 : double.Parse(Values.Rows[i]["Mobile2"].ToString());
Tel = Values.Rows[i]["Tel"].ToString() == "" ? 0 : double.Parse(Values.Rows[i]["Tel"].ToString());
Category = Values.Rows[i]["Category"].ToString();
Insert("INSERT INTO client(Mobile1,Mobile2,Tel,Category) VALUES(" + Mobile1 + "," + Mobile2 + "," + Tel + ",'" + Category + "')");
Msg.Text = "Submitied successfully to the server ";
}
}
}
You can try SqlBulkCopy to insert Datatable to Database Table
Something like this,
using (SqlBulkCopy bulkCopy = new SqlBulkCopy(sqlConnection, SqlBulkCopyOptions.KeepIdentity))
{
bulkCopy.DestinationTableName = DestTableName;
string[] DtColumnName = YourDataTableColumns;
foreach (string dbcol in DbColumnName)//To map Column of Datatable to that of DataBase tabele
{
foreach (string dtcol in DtColumnName)
{
if (dbcol.ToLower() == dtcol.ToLower())
{
SqlBulkCopyColumnMapping mapID = new SqlBulkCopyColumnMapping(dtcol, dbcol);
bulkCopy.ColumnMappings.Add(mapID);
break;
}
}
}
bulkCopy.WriteToServer(YourDataTableName.CreateDataReader());
bulkCopy.Close();
}
For more Read http://msdn.microsoft.com/en-us/library/system.data.sqlclient.sqlbulkcopy.aspx
You are inserting 1 row at a time, which is very expensive for this amount of data
In those cases you should use bulk insert, so the round trip to DB will be only once, if you need to roll back - all is the same transaction
You can use SqlBulkCopy which is more work, or you can use the batch update feature of the SqlAdpater. Instead of creating your own insert statement, then building a sqladapter, and then manually executing it, create a dataset, fill it, create one sqldataadpater, set the number of inserts in a batch, then execute the adapter once.
I could repeat the code, but this article shows exactly how to do it: http://msdn.microsoft.com/en-us/library/kbbwt18a%28v=vs.80%29.aspx
protected void SendToServer_Click(object sender, EventArgs e)
{
DataTable Values = Session["valuesdt"] as DataTable ;
if(Values.Rows.Count>0)
{
DataTable dv = Values.DefaultView.ToTable(true, "Mobile1", "Mobile2", "Tel", "Category");
//Fix up default values
for (int i = 0; i < Values.Rows.Count; i++)
{
Values.Rows[i]["Mobile1"] =Values.Rows[i]["Mobile1"].ToString()==""?0: double.Parse(Values.Rows[i]["Mobile1"].ToString());
Values.Rows[i]["Mobile2"] = Values.Rows[i]["Mobile2"].ToString() == "" ? 0 : double.Parse(Values.Rows[i]["Mobile2"].ToString());
Values.Rows[i]["Tel"] = Values.Rows[i]["Tel"].ToString() == "" ? 0 : double.Parse(Values.Rows[i]["Tel"].ToString());
Values.Rows[i]["Category"] = Values.Rows[i]["Category"].ToString();
}
BatchUpdate(dv,1000);
}
}
public static void BatchUpdate(DataTable dataTable,Int32 batchSize)
{
// Assumes GetConnectionString() returns a valid connection string.
string connectionString = GetConnectionString();
// Connect to the database.
using (SqlConnection connection = new SqlConnection(connectionString))
{
// Create a SqlDataAdapter.
SqlDataAdapter adapter = new SqlDataAdapter();
// Set the INSERT command and parameter.
adapter.InsertCommand = new SqlCommand(
"INSERT INTO client(Mobile1,Mobile2,Tel,Category) VALUES(#Mobile1,#Mobile2,#Tel,#Category);", connection);
adapter.InsertCommand.Parameters.Add("#Mobile1",
SqlDbType.Float);
adapter.InsertCommand.Parameters.Add("#Mobile2",
SqlDbType.Float);
adapter.InsertCommand.Parameters.Add("#Tel",
SqlDbType.Float);
adapter.InsertCommand.Parameters.Add("#Category",
SqlDbType.NVarchar, 50);
adapter.InsertCommand.UpdatedRowSource = UpdateRowSource.None;
// Set the batch size.
adapter.UpdateBatchSize = batchSize;
// Execute the update.
adapter.Update(dataTable);
}
}
I know this is a super old post, but you should not need to use the bulk operations explained in the existing answers for 5000 inserts. Your performance is suffering so much because you close and reopen the connection for each row insert. Here is some code I have used in the past that keeps one connection open and executes as many commands as needed to push all the data to the DB:
public static class DataWorker
{
public static Func<IEnumerable<T>, Task> GetStoredProcedureWorker<T>(Func<SqlConnection> connectionSource, string storedProcedureName, Func<T, IEnumerable<(string paramName, object paramValue)>> parameterizer)
{
if (connectionSource is null) throw new ArgumentNullException(nameof(connectionSource));
SqlConnection openConnection()
{
var conn = connectionSource() ?? throw new ArgumentNullException(nameof(connectionSource), $"Connection from {nameof(connectionSource)} cannot be null");
var connState = conn.State;
if (connState != ConnectionState.Open)
{
conn.Open();
}
return conn;
}
async Task DoStoredProcedureWork(IEnumerable<T> workData)
{
using (var connection = openConnection())
using (var command = connection.CreateCommand())
{
command.CommandType = CommandType.StoredProcedure;
command.CommandText = storedProcedureName;
command.Prepare();
foreach (var thing in workData)
{
command.Parameters.Clear();
foreach (var (paramName, paramValue) in parameterizer(thing))
{
command.Parameters.AddWithValue(paramName, paramValue ?? DBNull.Value);
}
await command.ExecuteNonQueryAsync().ConfigureAwait(false);
}
}
}
return DoStoredProcedureWork;
}
}
This was actually from a project where I was gathering emails for a restriction list, so kind of relevant example of what a parameterizer argument might look like and how to use the above code:
IEnumerable<(string,object)> RestrictionToParameter(EmailRestriction emailRestriction)
{
yield return ("#emailAddress", emailRestriction.Email);
yield return ("#reason", emailRestriction.Reason);
yield return ("#restrictionType", emailRestriction.RestrictionType);
yield return ("#dateTime", emailRestriction.Date);
}
var worker = DataWorker.GetStoredProcedureWorker<EmailRestriction>(ConnectionFactory, #"[emaildata].[AddRestrictedEmail]", RestrictionToParameter);
await worker(emailRestrictions).ConfigureAwait(false);
I am trying to insert huge multiple rows in a single transaction. But the SQL Server database is not getting reflected with the update. Below is my code snippet. What am I doing wrong?
private void Form1_Load(object sender, EventArgs e)
{
string conString = Properties.Settings.Default.Database1ConnectionString;
Database1DataSet.Table1DataTable dt = new Database1DataSet.Table1DataTable();
for (int i = 0; i < 5; i++)
{
dt.AddTable1Row(i.ToString(), "name" + i.ToString());
}
// Open the connection using the connection string.
using (SqlCeConnection con = new SqlCeConnection(conString))
{
try
{
var cmd = con.CreateCommand();
con.Open();
cmd.CommandType = CommandType.TableDirect;
cmd.CommandText = "Table1";
var rs = cmd.ExecuteResultSet(ResultSetOptions.Updatable)
var rec = rs.CreateRecord();
foreach (DataRow dr in dt.Rows)
{
rec.SetString(0, dr["ID"].ToString());
rec.SetString(1, dr["Name"].ToString());
rs.Insert(rec);
}
rs.Close();
rs.Dispose();
cmd.Dispose();
}
catch (Exception ex) { }
}
}
You are facing this: http://erikej.blogspot.com/2010/05/faq-why-does-my-changes-not-get-saved.html