I'm using .NET Connector to connect to MySQL.
In my application there are few threads using the same connection, so if a MySQLDataReader is not closed yet and some thread is trying execute a query it gives that error:
There is already an open DataReader associated with this Connection which must be closed first.
Will there ever be support in MySQL for multiple result sets or however it's called?
My database management class:
public class DatabaseConnection
{
private MySqlConnection conn;
public void Connect(string server, string user, string password, string database, int port = 3306)
{
string connStr = String.Format("server={0};user={1};database={2};port={3};password={4};charset=utf8",
server, user, database, port, password);
conn = new MySqlConnection(connStr);
conn.Open();
}
private MySqlCommand PrepareQuery(string query, object[] args)
{
MySqlCommand cmd = new MySqlCommand();
cmd.Connection = conn;
for (int i = 0; i < args.Length; i++)
{
string param = "{" + i + "}";
string paramName = "#DBVar_" + i;
query = query.Replace(param, paramName);
cmd.Parameters.AddWithValue(paramName, args[i]);
}
cmd.CommandText = query;
return cmd;
}
public List<Dictionary<string, object>> Query(string query, params object[] args)
{
MySqlCommand cmd = PrepareQuery(query, args);
MySqlDataReader reader = cmd.ExecuteReader();
List<Dictionary<string, object>> rows = new List<Dictionary<string, object>>();
while (reader.Read())
{
Dictionary<string, object> row = new Dictionary<string, object>();
for (int i = 0; i < reader.FieldCount; i++)
{
row.Add(reader.GetName(i), reader.GetValue(i));
}
rows.Add(row);
}
reader.Close();
return rows;
}
public object ScalarQuery(string query, params object[] args)
{
MySqlCommand cmd = PrepareQuery(query, args);
return cmd.ExecuteScalar();
}
public void Execute(string query, params object[] args)
{
MySqlCommand cmd = PrepareQuery(query, args);
cmd.ExecuteNonQuery();
}
public void Close()
{
conn.Close();
}
}
An example of how I use this:
DatabaseConnection Conn = new DatabaseConnection();
Conn.Connect("localhost", "root", "", "foogle");
var rows = conn.Query("SELECT * FROM `posts` WHERE `id` = {0}", postID);
foreach (var row in rows)
{
Console.WriteLine(row["title"]); // Writes the post's title (example)
}
Multiple result sets refers to a single query or query batch returning multiple row sets. Those results are accessed through the one and only DataReader for that connection.
What you're asking for is something quite different. You need the ability to performing multiple simultaneous queries of a single connection. Afaik .NET does not support that, not for SQL Server or any other driver.
Sharing a connection between multiple threads is a bad idea and totally unnecessary. .NET will use a connection pool to limit the total number of connections so It's perfectly safe to get a new connection for each (set of) queries you want to execute. Limit the scope of a connection to a thread and your problem will go away.
Related
I have to write identical tables to two different Sqlite database. The tables rewrite existing data and ids, so I currently just delete the entire table and rewrite. Depending on the order that I write to the db, the C# SQLiteCommandBuilder does not update the second db called.
If I call Db_A before Db_B, then A gets written and B gets deleted and vice versa. Can anyone tell me why the second table enters the code to get deleted, but the Sqlite adapter never updates the second table? It doesn't throw an error either.
public static bool WriteDt_Name(DataTable dt)
{
using (connA = GetDbAConn())
{
SaveDataTable(connA, dt);
}
using (connB = GetDbBConn())
{
SaveDataTable(connB, dt);
}
return true;
}
public static void SaveDataTable(SQLiteConnection conn, DataTable dt)
{
string table = dt.TableName;
var cmd = conn.CreateCommand();
cmd.CommandText = string.Format("DELETE FROM {0}", table);
int val = cmd.ExecuteNonQuery();
cmd.CommandText = string.Format("SELECT * FROM {0}", table);
using (var adapter = new SQLiteDataAdapter(cmd))
{
using (SQLiteCommandBuilder builder = new SQLiteCommandBuilder(adapter))
{
adapter.Update(dt);
conn.Close();
}
}
}
public static SQLiteConnection GetDbAConn()
{
string base_dir = System.AppDomain.CurrentDomain.BaseDirectory;
string path = Directory.GetParent(Directory.GetParent(Directory.GetParent(Directory.GetParent(Directory.GetParent(base_dir).ToString()).ToString()).ToString()).ToString()).ToString();
path = path + "\\db\\DbA.sqlite;";
SQLiteConnection conn = new SQLiteConnection("Data Source=" + path + "Version=3;");
return conn;
}
I have tried splitting SaveDataTable into a SaveDt_A and SaveDt_B and calling it that way. I still get the same result.
I have a problem with understanding transactions handling in System.Data.Sqlite connected with the following piece of code.
public static string ConnenctionString = "Data Source=C:\\DB\\Test.db";
public static void Test1()
{
SQLiteConnection connection = new SQLiteConnection(ConnenctionString);
connection.Open();
SQLiteTransaction transaction = connection.BeginTransaction();
for (int i = 0; i < 1000000; i++)
{
string sql = $"insert into table1 (id) values ({i})";
SQLiteCommand command = new SQLiteCommand(sql, connection);
command.ExecuteNonQuery();
}
transaction.Commit();
}
public static void Test2()
{
SQLiteConnection connection = new SQLiteConnection(ConnenctionString);
connection.Open();
string sql = "select count(*) from table1";
SQLiteCommand command = new SQLiteCommand(sql, connection);
SQLiteDataReader reader = command.ExecuteReader();
reader.Read();
Console.WriteLine(reader[0]);
}
public static void Test3()
{
for (int i = 0; i < 300; i++)
{
SQLiteConnection connection = new SQLiteConnection(ConnenctionString);
connection.Open();
SQLiteTransaction transaction = connection.BeginTransaction();
string sql = $"insert into table1 (id) values ({i})";
SQLiteCommand command = new SQLiteCommand(sql, connection);
command.ExecuteNonQuery();
transaction.Commit();
}
}
static void Main(string[] args)
{
//Everything is ok
Task.Factory.StartNew(() => Test1());
Thread.Sleep(1000);
Task.Factory.StartNew(() => Test2());
//Exception db is locked
Task.Factory.StartNew(() => Test3());
Thread.Sleep(1000);
Task.Factory.StartNew(() => Test2());
Console.ReadLine();
}
In one task I am executing some long running operation (in one transaction) - method Test1. In second task, I am trying to execute another query, without transaction - method Test2. Everything is running OK, I obtain a result that doesn't include changes from Test1 - because transaction hasn't been commited yet.
But when trying to execute in one task lots of queries - each in separate transaction, like in method Test3 and trying meanwhile to execute method Test2, I am obtaing exception: DB is locked. Is there a way to omit such behaviour modyfing connection string?
I would recommend using using statements (unfortunate pun):
using (var connection = new SqliteConnection("conn-string"))
{
connection.Open();
using (var transaction = connection.BeginTransaction())
{
for (int i = 0; i < 1000000; i++)
{
string sql = $"insert into table1 (id) values ({i})";
using (var command = new SqliteCommand(sql, connection))
{
command.ExecuteNonQuery();
}
}
transaction.Commit();
}
}
You can try to modify your database so it will use write-ahead logging. It will allow simultaneous access for multiple tasks without locking.
ExecuteQuery("pragma journal_mode=WAL");
immediately after creating the database. You need to execute this statement only once - it's persistent.
I am trying to doing this:
Read a row from an SQLite db (in GetRuleByID() method)
Update the same row that I just read during (1) (See UpdateStatusForRuleID() method)
However my problem is that SQLite locks the database after the SELECT in GetRuleByID() so that update in UpdateStatusForRuleID() is only successful when called the first time.
I have tried enabling Write-Ahead-Logging in SQLite as well as PRAGMA read_uncommitted=1 in order to avoid SQLite locking the database for the SELECT, but this does not appear to work.
This should be simple but I have so far spent a complete night trying to solve this... Please help !
private static MicroRuleEngine.Rule GetRuleByID(int ruleID, SQLiteConnection connection, out Dictionary<string, string> dict)
{
dict = new Dictionary<string, string>();
string sql = String.Format("select * from rules WHERE ID = {0} ", ruleID.ToString());
SQLiteCommand command = new SQLiteCommand(sql, connection);
SQLiteDataReader reader = command.ExecuteReader();
if (reader.HasRows)
{
reader.Read();
// Convert row into a dictionary
for (int lp = 0; lp < reader.FieldCount; lp++)
{
dict.Add(reader.GetName(lp), reader.GetValue(lp) as string);
}
string json = dict["fulljson"];
MicroRuleEngine.Rule r = Newtonsoft.Json.JsonConvert.DeserializeObject<MicroRuleEngine.Rule>(json);
//command.Dispose();
return r;
}
}
internal static void UpdateStatusForRuleID(SQLConnectionManager DBMANAGER, int ruleID, bool status)
{
Dictionary<string, string> dict = null;
string dbVal = (status) ? "1" : "0";
MicroRuleEngine.Rule r = null;
string newJSON = null;
using (SQLiteConnection connection = DBMANAGER.CreateConnection())
{
r = GetRuleByID(ruleID, connection, out dict);
r.Active = (status);
newJSON = Newtonsoft.Json.JsonConvert.SerializeObject(r);
Thread.Sleep(1000);
string sql = "UPDATE rules SET active = #a, fulljson=#j WHERE ID = #i";
using (var command = new SQLiteCommand(sql, connection))
{
command.Parameters.Add(new SQLiteParameter("#a", dbVal));
command.Parameters.Add(new SQLiteParameter("#i", ruleID));
command.Parameters.Add(new SQLiteParameter("#j", newJSON));
command.ExecuteNonQuery(); // Database is locked here ???
}
connection.Close();
}
}
"Database is locked" means that some other connection (in the same or some other process) still has an active transaction.
You don't need multiple connections (unless you are using multiple threads); just use a single connection object for all database accesses.
Ensure that all command, reader, and transaction objects (and connections, if you decide to use temporary ones) are properly cleaned up, by using using:
using (var command = new SQLiteCommand(sql, connection))
using (var reader = command.ExecuteReader())
{
if (reader.HasRows)
...
}
Apparently, the code below works. I basically dropped the GetRuleByID() method (but then I had to re-write 4 other methods)
Thanks to all who provided input.
internal static void UpdateStatusForRuleID(SQLConnectionManager DBMANAGER, int ruleID, bool status)
{
string dbVal = (status) ? "1" : "0";
MicroRuleEngine.Rule r = null;
string newJSON = null;
using (SQLiteConnection conn = DBMANAGER.CreateConnection())
{
string sql = String.Format("select * from rules WHERE ID = {0} ", ruleID.ToString());
using (var command = new SQLiteCommand(sql, conn))
using (var reader = command.ExecuteReader())
{
if (reader.HasRows)
{
reader.Read();
string json = reader["fulljson"].ToString();
r = Newtonsoft.Json.JsonConvert.DeserializeObject<MicroRuleEngine.Rule>(json);
r.Active = (status);
newJSON = Newtonsoft.Json.JsonConvert.SerializeObject(r);
string sql2 = "UPDATE rules SET active = #a, fulljson=#j WHERE ID = #i";
using (var command2 = new SQLiteCommand(sql2, conn))
{
command2.Parameters.Add(new SQLiteParameter("#a", dbVal));
command2.Parameters.Add(new SQLiteParameter("#i", ruleID));
command2.Parameters.Add(new SQLiteParameter("#j", newJSON));
command2.ExecuteNonQuery();
}
}
}
}
}
Im trying to build up a little status-tool. I need to get results of multiple queries (about 4-5). The general connection-setup and 'how-to-read-data' is already done but I cant figure out how the another query executed.
Everything I found while searching for it is for the SqlClient. Im totally overcharged with this.
Here is my code so far (be patient, im a newbie to this):
private void button1_Click(object sender, EventArgs e)
{
if(listView1.Items.Count > 1)
{
listView1.Items.Clear();
}
var listMember = new List<string>{};
var listOnline = new List<string>{};
// SQL PART //
string connString = "Server=10*****;Port=3306;Database=e***;Uid=e***;password=********************;";
MySqlConnection conn = new MySqlConnection(connString);
MySqlCommand command = conn.CreateCommand();
command.CommandText = "SELECT fullname,online FROM member WHERE active = '1' ORDER BY online DESC";
try
{
conn.Open();
}
catch (Exception ex)
{
listView1.Items.Add("Error: " + ex);
}
MySqlDataReader reader = command.ExecuteReader();
while(reader.Read())
{
listMember.Add(reader["fullname"].ToString());
listOnline.Add(reader["online"].ToString());
}
conn.Close();
// SQL ENDING //
// SET ENTRIES TO LISTVIEW //
int counter = 0;
foreach(string member in listMember)
{
ListViewItem item = new ListViewItem(new[] { member, listOnline.ElementAt(counter) });
item.ForeColor = Color.Green;
listView1.Items.Add(item);
counter++;
}
}
Im not really sure how the design/layout will look like in the end, so I would like to just append the results to lists in the sql-part to process the data later out of the lists.
Do I really have to setup a complete new connection after conn.Close()? Or is there any other way? I can just imagine: 5 queries with their own connection,try,catch and 2 loops... this will get about 100-200 lines just for getting the results out of 5 queries. Isnt that a bit too much for such an easy thing?
Hope for some help.
Greetings.
According to the new comments my latest code:
Top:
public partial class Form1 : Form
{
public static string connString = "Server=10****;Port=3306;Database=e****;Uid=e****;password=****;";
public Form1()
{
InitializeComponent();
MySqlConnection conn = new MySqlConnection(connString); // Error gone!
}
Body part:
public void QueryTwoFields(string s, List<string> S1, List<string> S2)
{
try
{
MySqlCommand cmd = conn.CreateCommand(); // ERROR: conn does not exist in the current context.
cmd.CommandType = CommandType.Text;
string command = s;
cmd.CommandText = command;
MySqlDataReader sqlreader = cmd.ExecuteReader();
while (sqlreader.Read())
{
S1.Add(sqlreader[0].ToString());
S2.Add(sqlreader[1].ToString());
}
sqlreader.Close();
}
catch (Exception ex)
{
MessageBox.Show(ex.ToString());
}
}
private void button1_Click(object sender, EventArgs e)
{
if(listView1.Items.Count > 1)
{
listView1.Items.Clear();
}
var listMember = new List<string>{};
var listOnline = new List<string>{};
using (conn) // ERROR: conn does not exist in the current context.
{
conn.Open();
///...1st Query
QueryTwoFields("SELECT fullname,online FROM member WHERE active = '1' ORDER BY online DESC",listMember,listOnline);
//...2nd query
//QueryTwoFields("your new Select Statement", otherList, otherList);
}
}
You don't have to close connection every time you execute one query rarher than close the sqlreader assigned to that connection. Finally when all of your queries have been executed you close the connection. Consider also the use of using:
You cal also define a method for execution your Query in order for your code not to be repetive:
public void QueryTwoFields(string s, List<string> S1, List<string> S2)
///Select into List S1 and List S2 from Database (2 fields)
{
try
{
MySqlCommand cmd = conn.CreateCommand();
cmd.CommandType = CommandType.Text;
string command = s;
cmd.CommandText = command;
MySqlDataReader sqlreader = cmd.ExecuteReader();
while (sqlreader.Read())
{
S1.Add(sqlreader[0].ToString());
S2.Add(sqlreader[1].ToString());
}
sqlreader.Close();
}
catch (Exception ex)
{
MessageBox.Show(ex.ToString());
}
}
private void button1_Click(object sender, EventArgs e)
{
if(listView1.Items.Count > 1)
{
listView1.Items.Clear();
}
var listMember = new List<string>{};
var listOnline = new List<string>{};
// SQL PART //
using (conn)
{
conn.Open();
///...1st Query
QueryTwoFields("SELECT fullname,online FROM member WHERE active = '1' ORDER BY online DESC",listmember,listonline)
//...2nd query
QueryTwoFields("your new Select Statement",myOtherList1,myOtherlist2)
....
}
}
EDIT :
Take in mind you cant define QueryTwoFields method inside button handler. You must define it outside (see code above).
Also Define your connection data in the start of the programm:
namespace MyProject
{
/// <summary>
/// Defiine your connectionstring and connection
/// </summary>
///
public partial class Form1 : Form
{ public static string connString = "Server=10*****;Port=3306;Database=e***;Uid=e***;password=********************;";
MySqlConnection conn = new MySqlConnection(connString);
.........
Datatables are fantastic
Using a data table is a nice way to do both read and write. And it comes with the luxury of eveything you can do with a datatable - like asssigning it directly to a datagrid control, sorting, selecting and deleting while disconnected.
The sample below assumes a MySqlConnection conection property managed by calls to your own OpenConnection() and CloseConnection() methods not shown.
Simple datatable read demo:
public DataTable Select(string query = "")
{
//Typical sql: "SELECT * FROM motorparameter"
DataTable dt = new DataTable();
//Open connection
if (this.OpenConnection() == true)
{
//Create Command
MySqlCommand cmd = new MySqlCommand(query, connection);
//Create a data reader and Execute the command
MySqlDataReader dataReader = cmd.ExecuteReader();
dt.Load(dataReader);
//close Data Reader
dataReader.Close();
//close Connection
this.CloseConnection();
//return data table
return dt;
}
else
{
return dt;
}
}
In case of writing back the datatable to the database - supply the SQL you used in the read (or would have used to read to the data table):
public void Save(DataTable dt, string DataTableSqlSelect)
{
//Typically "SELECT * FROM motorparameter"
string query = DataTableSqlSelect;
//Open connection
if (this.OpenConnection() == true)
{
//Create Command
MySqlCommand mySqlCmd = new MySqlCommand(query, connection);
MySqlDataAdapter adapter = new MySqlDataAdapter(mySqlCmd);
MySqlCommandBuilder myCB = new MySqlCommandBuilder(adapter);
adapter.UpdateCommand = myCB.GetUpdateCommand();
adapter.Update(dt);
//close Connection
this.CloseConnection();
}
else
{
}
}
The neat thing the datatable is extremely flexible. You can run your own selects against the table once it contains data and before writing back you can set or reset what rows needs updating and by default the datatable keeps track of what rows you update in the table. Do not forget primary key column(s) for all tables in the db.
For multiple queries consider if possible using a join between the database tables or same table if data related or use a UNION sql syntax if column count and type of data is the same. You can allways "create" your extra column in the select to differ what data comes from what part of the UNION.
Also consider using CASE WHEN sql syntax to conditionally select data from different sources.
in my code the user can upload an excel document wish contains it's phone contact list.Me as a developer should read that excel file turn it into a dataTable and insert it into the database .
The Problem is that some clients have a huge amount of contacts like saying 5000 and more contacts and when i am trying to insert this amount of data into the database it's crashing and giving me a timeout exception.
What would be the best way to avoid this kind of exception and is their any code that can reduce the time of the insert statement so the user don't wait too long ?
the code
public SqlConnection connection = new SqlConnection(System.Configuration.ConfigurationManager.ConnectionStrings["ConnectionString"].ConnectionString);
public void Insert(string InsertQuery)
{
SqlDataAdapter adp = new SqlDataAdapter();
adp.InsertCommand = new SqlCommand(InsertQuery, connection);
if (connection.State == System.Data.ConnectionState.Closed)
{
connection.Open();
}
adp.InsertCommand.ExecuteNonQuery();
connection.Close();
}
protected void submit_Click(object sender, EventArgs e)
{
string UploadFolder = "Savedfiles/";
if (Upload.HasFile) {
string fileName = Upload.PostedFile.FileName;
string path=Server.MapPath(UploadFolder+fileName);
Upload.SaveAs(path);
Msg.Text = "successfully uploaded";
DataTable ValuesDt = new DataTable();
ValuesDt = ConvertExcelFileToDataTable(path);
Session["valuesdt"] = ValuesDt;
Excel_grd.DataSource = ValuesDt;
Excel_grd.DataBind();
}
}
protected void SendToServer_Click(object sender, EventArgs e)
{
DataTable Values = Session["valuesdt"] as DataTable ;
if(Values.Rows.Count>0)
{
DataTable dv = Values.DefaultView.ToTable(true, "Mobile1", "Mobile2", "Tel", "Category");
double Mobile1,Mobile2,Tel;string Category="";
for (int i = 0; i < Values.Rows.Count; i++)
{
Mobile1 =Values.Rows[i]["Mobile1"].ToString()==""?0: double.Parse(Values.Rows[i]["Mobile1"].ToString());
Mobile2 = Values.Rows[i]["Mobile2"].ToString() == "" ? 0 : double.Parse(Values.Rows[i]["Mobile2"].ToString());
Tel = Values.Rows[i]["Tel"].ToString() == "" ? 0 : double.Parse(Values.Rows[i]["Tel"].ToString());
Category = Values.Rows[i]["Category"].ToString();
Insert("INSERT INTO client(Mobile1,Mobile2,Tel,Category) VALUES(" + Mobile1 + "," + Mobile2 + "," + Tel + ",'" + Category + "')");
Msg.Text = "Submitied successfully to the server ";
}
}
}
You can try SqlBulkCopy to insert Datatable to Database Table
Something like this,
using (SqlBulkCopy bulkCopy = new SqlBulkCopy(sqlConnection, SqlBulkCopyOptions.KeepIdentity))
{
bulkCopy.DestinationTableName = DestTableName;
string[] DtColumnName = YourDataTableColumns;
foreach (string dbcol in DbColumnName)//To map Column of Datatable to that of DataBase tabele
{
foreach (string dtcol in DtColumnName)
{
if (dbcol.ToLower() == dtcol.ToLower())
{
SqlBulkCopyColumnMapping mapID = new SqlBulkCopyColumnMapping(dtcol, dbcol);
bulkCopy.ColumnMappings.Add(mapID);
break;
}
}
}
bulkCopy.WriteToServer(YourDataTableName.CreateDataReader());
bulkCopy.Close();
}
For more Read http://msdn.microsoft.com/en-us/library/system.data.sqlclient.sqlbulkcopy.aspx
You are inserting 1 row at a time, which is very expensive for this amount of data
In those cases you should use bulk insert, so the round trip to DB will be only once, if you need to roll back - all is the same transaction
You can use SqlBulkCopy which is more work, or you can use the batch update feature of the SqlAdpater. Instead of creating your own insert statement, then building a sqladapter, and then manually executing it, create a dataset, fill it, create one sqldataadpater, set the number of inserts in a batch, then execute the adapter once.
I could repeat the code, but this article shows exactly how to do it: http://msdn.microsoft.com/en-us/library/kbbwt18a%28v=vs.80%29.aspx
protected void SendToServer_Click(object sender, EventArgs e)
{
DataTable Values = Session["valuesdt"] as DataTable ;
if(Values.Rows.Count>0)
{
DataTable dv = Values.DefaultView.ToTable(true, "Mobile1", "Mobile2", "Tel", "Category");
//Fix up default values
for (int i = 0; i < Values.Rows.Count; i++)
{
Values.Rows[i]["Mobile1"] =Values.Rows[i]["Mobile1"].ToString()==""?0: double.Parse(Values.Rows[i]["Mobile1"].ToString());
Values.Rows[i]["Mobile2"] = Values.Rows[i]["Mobile2"].ToString() == "" ? 0 : double.Parse(Values.Rows[i]["Mobile2"].ToString());
Values.Rows[i]["Tel"] = Values.Rows[i]["Tel"].ToString() == "" ? 0 : double.Parse(Values.Rows[i]["Tel"].ToString());
Values.Rows[i]["Category"] = Values.Rows[i]["Category"].ToString();
}
BatchUpdate(dv,1000);
}
}
public static void BatchUpdate(DataTable dataTable,Int32 batchSize)
{
// Assumes GetConnectionString() returns a valid connection string.
string connectionString = GetConnectionString();
// Connect to the database.
using (SqlConnection connection = new SqlConnection(connectionString))
{
// Create a SqlDataAdapter.
SqlDataAdapter adapter = new SqlDataAdapter();
// Set the INSERT command and parameter.
adapter.InsertCommand = new SqlCommand(
"INSERT INTO client(Mobile1,Mobile2,Tel,Category) VALUES(#Mobile1,#Mobile2,#Tel,#Category);", connection);
adapter.InsertCommand.Parameters.Add("#Mobile1",
SqlDbType.Float);
adapter.InsertCommand.Parameters.Add("#Mobile2",
SqlDbType.Float);
adapter.InsertCommand.Parameters.Add("#Tel",
SqlDbType.Float);
adapter.InsertCommand.Parameters.Add("#Category",
SqlDbType.NVarchar, 50);
adapter.InsertCommand.UpdatedRowSource = UpdateRowSource.None;
// Set the batch size.
adapter.UpdateBatchSize = batchSize;
// Execute the update.
adapter.Update(dataTable);
}
}
I know this is a super old post, but you should not need to use the bulk operations explained in the existing answers for 5000 inserts. Your performance is suffering so much because you close and reopen the connection for each row insert. Here is some code I have used in the past that keeps one connection open and executes as many commands as needed to push all the data to the DB:
public static class DataWorker
{
public static Func<IEnumerable<T>, Task> GetStoredProcedureWorker<T>(Func<SqlConnection> connectionSource, string storedProcedureName, Func<T, IEnumerable<(string paramName, object paramValue)>> parameterizer)
{
if (connectionSource is null) throw new ArgumentNullException(nameof(connectionSource));
SqlConnection openConnection()
{
var conn = connectionSource() ?? throw new ArgumentNullException(nameof(connectionSource), $"Connection from {nameof(connectionSource)} cannot be null");
var connState = conn.State;
if (connState != ConnectionState.Open)
{
conn.Open();
}
return conn;
}
async Task DoStoredProcedureWork(IEnumerable<T> workData)
{
using (var connection = openConnection())
using (var command = connection.CreateCommand())
{
command.CommandType = CommandType.StoredProcedure;
command.CommandText = storedProcedureName;
command.Prepare();
foreach (var thing in workData)
{
command.Parameters.Clear();
foreach (var (paramName, paramValue) in parameterizer(thing))
{
command.Parameters.AddWithValue(paramName, paramValue ?? DBNull.Value);
}
await command.ExecuteNonQueryAsync().ConfigureAwait(false);
}
}
}
return DoStoredProcedureWork;
}
}
This was actually from a project where I was gathering emails for a restriction list, so kind of relevant example of what a parameterizer argument might look like and how to use the above code:
IEnumerable<(string,object)> RestrictionToParameter(EmailRestriction emailRestriction)
{
yield return ("#emailAddress", emailRestriction.Email);
yield return ("#reason", emailRestriction.Reason);
yield return ("#restrictionType", emailRestriction.RestrictionType);
yield return ("#dateTime", emailRestriction.Date);
}
var worker = DataWorker.GetStoredProcedureWorker<EmailRestriction>(ConnectionFactory, #"[emaildata].[AddRestrictedEmail]", RestrictionToParameter);
await worker(emailRestrictions).ConfigureAwait(false);