I have written to append functions that insert data from custom c# list into MSAccess.
The first simply sets up a new connection for each individual recordset:
public static void appenddatatotable(string connectionstring, string tablename, string[] values)
{
var myconn = new OleDbConnection(connectionstring);
var cmd = new OleDbCommand();
cmd.CommandText = "INSERT INTO " + tablename + " ([RunDate],[ReportingGroup], [Tariff], [Year]) VALUES(#RunDate, #ReportingGroup, #Tariff, #Year)";
cmd.Parameters.AddRange(new[] { new OleDbParameter("#RunDate", values[0]), new OleDbParameter("#ReportingGroup", values[1]), new OleDbParameter("#Tariff", values[2]), new OleDbParameter("#Year", values[3])});
cmd.Connection = myconn;
myconn.Open();
cmd.ExecuteNonQuery();
myconn.Close();
}
I then simply loop over my list of values and call this function on each iteration. This works fine but is slow.
In the second function I tried to include the loop in the function and work with BeginTransction and Committransaction:
public static void appenddatatotable2(string connectionstring, string tablename, string datstr, List<PowRes> values)
{
var myconn = new OleDbConnection(connectionstring);
int icounter = 0;
var cmd = new OleDbCommand();
OleDbTransaction trans = null;
cmd.Connection = myconn;
myconn.Open();
foreach (var item in values)
{
if (icounter == 0)
{
trans = cmd.Connection.BeginTransaction();
cmd.Transaction = trans;
}
cmd.CommandText = "INSERT INTO " + tablename + " ([RunDate],[ReportingGroup], [Tariff], [Year]) VALUES(#RunDate, #ReportingGroup, #Tariff, #Year)";
if (string.IsNullOrEmpty(item.yr))
item.yr = "";
cmd.Parameters.AddRange(new[] { new OleDbParameter("#RunDate", datstr), new OleDbParameter("#ReportingGroup", item.RG), new OleDbParameter("#Tariff", item.tar), new OleDbParameter("#Year", item.yr)});
cmd.ExecuteNonQuery();
icounter++;
if (icounter >= 500)
{
trans.Commit();
icounter = 0;
}
}
if (icounter > 0)
{
trans.Commit();
}
myconn.Close();
}
This also works fine but is EVEN slower.
Is my code wrong? How could I speed up the multiple inserts?
Thanks!
did not test, just my guess for your second function: you add too many parameters to the same command over the loop - cmd.Parameters were never cleared before each usage..
normally committing large set of commands within one connection is much faster than doing them one by one at single connection.
another way to speed up your inserts is to dump all your insert statements into a long text, separated with semicolon, and then fire a commit in one go (i am not sure whether msAccess supports it or not)
EDIT:
to combine the update command into one text:
var updates = values.Select(x => string.Format("INSERT INTO myTable ([RunDate],[ReportingGroup], [Tariff], [Year]) VALUES({0}, {1}, {2}, {3})",
datstr, x.RG, x.tar, x.yr))
.Aggregate((m, n) => m + ";" + n);
cmd.CommandText = update;
Though this could have sql injection issues.
this should be significantly faster than all of your exiting versions
public static void appenddatatotable2(string connectionstring, string tablename, string datstr, List<PowRes> values)
{
string commandText = "INSERT INTO " + tablename + " ([RunDate],[ReportingGroup], [Tariff], [Year]) VALUES(#RunDate, #ReportingGroup, #Tariff, #Year)";
using (var myconn = new OleDbConnection(connectionstring))
{
myconn.Open();
using (var cmd = new OleDbCommand())
{
foreach (var item in values)
{
cmd.CommandText = commandText;
cmd.Parameters.Clear();
cmd.Parameters.AddRange(new[] { new OleDbParameter("#RunDate", datstr), new OleDbParameter("#ReportingGroup", item.RG), new OleDbParameter("#Tariff", item.tar), new OleDbParameter("#Year", item.yr) });
cmd.Connection = myconn;
cmd.Prepare();
cmd.ExecuteNonQuery();
}
}
}
}
Related
I have code using SqlBulkCopy to clone a lot of tables, it used to work before, but very weird, recently got exception
Received an invalid column length from the bcp client for colid
I have search this exception and still not solve my problem.
sqlBulkCopy.WriteToServer(reader) will raise this exception if a table has two continous columns which are both of type Char(1) or nvarchar(nn), and both have NULL. Sometime, changing the SqlBulkCopy.BatchSize makes it work, but many times, it will not.
After simplify, I have test case as follow, and it is reproduceable on two servers:
Create a table like below: (tested on SQL Server 2012 SP 4 and SQL Server 2016 SP2)
IF OBJECT_ID('dbo.TestTable', 'U') IS NOT NULL
DROP TABLE dbo.TestTable;
CREATE TABLE [dbo].[TestTable]
(
[value2] [char](1) NULL,
[value1] [char](1) NULL
) ON [PRIMARY]
GO
DECLARE #i int = 0
WHILE #i < 262
BEGIN
SET #i = #i + 1
INSERT INTO [dbo].[TestTable]([value2], [value1])
VALUES (null, null)
END
C# console (.net framework 4.7) code as below
class Program
{
// [change here]
static string sourceConn = #"Server={YourServer};Database={YourDatabase};User ID={userYourName};Password={yourPassword};connect timeout=15";
static void Main(string[] args)
{
CopyTable(sourceConn, sourceConn, "TestTable", "testTableBAK");
Console.ReadLine();
}
static void CopyTable(string sConnSource, string sConnDest, string sTableSource, string sTableDest)
{
if (IsTableExist(sConnDest, sTableDest))
{
RunNonQuerySQL(sConnDest, "DROP TABLE " + sTableDest);
Console.WriteLine($"existing table {sTableDest} dropped");
}
CopySchema(sConnDest, sTableSource, sTableDest);
using (SqlConnection connSource = new SqlConnection(sConnSource))
{
connSource.Open();
SqlCommand cmd = new SqlCommand();
cmd.Connection = connSource;
cmd.CommandText = "SELECT * FROM " + sTableSource;
// using (SqlBulkCopy sqlBulkCopy = new SqlBulkCopy(sConnDest, SqlBulkCopyOptions.KeepNulls | SqlBulkCopyOptions.KeepIdentity))
using (SqlBulkCopy sqlBulkCopy = new SqlBulkCopy(sConnDest))
{
// sqlBulkCopy.BatchSize = 1380; // this optional setting will work if set value smaller than 1397 for testTable on my new server (SQL server 13.0.5102.14)
// sqlBulkCopy.BatchSize = 261; // this optional setting will work if set value smaller than 261 for testTable on 2 older server (SQL server 11.0.7001)
sqlBulkCopy.DestinationTableName = sTableDest;
SqlDataReader reader = cmd.ExecuteReader();
try
{
// exception here
sqlBulkCopy.WriteToServer(reader);
Console.WriteLine("table copied");
}
catch (SqlException ex)
{
Console.WriteLine(ex.Message);
}
sqlBulkCopy.Close();
}
}
}
static bool IsTableExist(string sConn, string sTableName)
{
bool result = false;
using (SqlConnection conn = new SqlConnection(sConn))
{
conn.Open();
SqlCommand cmd = new SqlCommand();
string[] s = sTableName.Split('.');
if (s.Length > 1)
{
cmd.CommandText = "select count (*) as counter from information_schema.tables where table_name = '" + s[1] + "' and TABLE_SCHEMA='" + s[0] + "'";
}
else
{
cmd.CommandText = "select count (*) as counter from information_schema.tables where table_name = '" + sTableName + "'";
}
cmd.Connection = conn;
var count = Convert.ToInt32(cmd.ExecuteScalar());
result = count > 0;
}
return result;
}
static bool RunNonQuerySQL(string sConn, string sSQL)
{
bool result = false;
using (SqlConnection conn = new SqlConnection(sConn))
{
conn.Open();
SqlCommand cmd = new SqlCommand();
cmd.CommandText = sSQL;
cmd.Connection = conn;
var count = cmd.ExecuteNonQuery();
result = true;
}
return result;
}
static public bool CopySchema(string sConn, string sTableSource, string sTableDest)
{
return RunQuerySQL(sConn, "select * into " + sTableDest + " from " + sTableSource + " where 1=2");
}
static public bool RunQuerySQL(string sConn, string sSQL)
{
using (SqlConnection conn = new SqlConnection(sConn))
{
conn.Open();
SqlCommand cmd = new SqlCommand();
cmd.CommandText = sSQL;
cmd.Connection = conn;
SqlDataReader reader = cmd.ExecuteReader();
if (reader.Read())
{
return true;
}
else
{
return false;
}
}
}
}
I just experienced this error.
I tried to insert a string with a lenght of 5 into a table column with a definition of "varchar(4)".
The error message in my case was:
"Received an invalid column length from the bcp client for colid 2".
"Colid 2" refered to the second column of the row (DataRow) that was part of the DataTable which I used as parameter for the call to the SqlBulkCopy.WriteToServer(DataTable table) method.
The solution in my case was to add validation code that checks the lenght of the strings in my input data before trying to call SqlBulkCopy.WriteToServer().
public void addintovisitor()
{
string companyname = (txtvisitor.Text.ToUpper());
DataSet result = new DataSet();
visitorcompany vc = new visitorcompany();
string Location1 = Convert.ToString(Session["location"]);
vc.checksamecompanyname(ref result, Location1);
for (int i = 0; i < result.Tables["details"].Rows.Count; i++)
{
if (companyname == result.Tables["details"].Rows[i]["Companyname"].ToString())
{
}
else
{
string strConn = Convert.ToString(ConfigurationManager.ConnectionStrings["connectionstring"]);
SqlConnection conn = new SqlConnection(strConn);
SqlCommand cmd = new SqlCommand(
"INSERT INTO tblVisitorcompany ([CompanyName], " +
"[Location1]) " +
"VALUES(#CompanyName, #Location1)", conn);
cmd.Parameters.AddWithValue("#Companyname", companyname);
cmd.Parameters.AddWithValue("#Location1", Location1);
conn.Open();
cmd.ExecuteNonQuery();
conn.Close();
}
}
}
My visitorcompany class:
public int checksamecompanyname(ref DataSet result, string Location1)
{
string strConn = Convert.ToString(
ConfigurationManager.ConnectionStrings
["connectionstring"]);
SqlConnection conn = new SqlConnection(strConn);
SqlCommand cmd = new SqlCommand
("select Companyname from tblVisitorcompany where Location1 ='" + Location1 + "'", conn);
SqlDataAdapter da = new SqlDataAdapter(cmd);
conn.Open();
da.Fill(result, "details");
conn.Close();
//Return 0 when no error occurs.
return 0;
}
I am trying to search one row at a time to check whether the sql table got the same companyname. if there is already exisiting companyname, the program will do nothing. If this is a new companyname, the program will add companyname into the sql table. However, when adding new companyname, the program will add more than once. Can someone please help me to re-edit my program such that it only add one new companyname. Many thanks.
using( var connection = new SqlConnection( "my connection string" ) ) {
using( var command = connection.CreateCommand() ) {
command.CommandText = "SELECT Column1, Column2, Column3 FROM myTable";
connection.Open();
using( var reader = command.ExecuteReader() ) {
var indexOfColumn1 = reader.GetOrdinal( "Column1" );
var indexOfColumn2 = reader.GetOrdinal( "Column2" );
var indexOfColumn3 = reader.GetOrdinal( "Column3" );
while( reader.Read() ) {
var value1 = reader.GetValue( indexOfColumn1 );
var value2 = reader.GetValue( indexOfColumn2 );
var value3 = reader.GetValue( indexOfColumn3 );
// now, do something what you want
}
}
connection.Close();
}
dont use companyname as an argument of your insert command, since it is stays the same in for loop. Use result.Tables["details"].Rows[i]["Companyname"].ToString() instead:
...
cmd.Parameters.AddWithValue("#Companyname", result.Tables["details"].Rows[i]["Companyname"].ToString());
...
Check if the value exists, then add it if not.
A simple change in your code:
bool valueFound = false;
// check if the value exists
for (int i = 0; i < result.Tables["details"].Rows.Count; i++)
{
if (companyname == result.Tables["details"].Rows[i]["Companyname"].ToString())
{
// it exists so we exit the loop
valueFound = true;
break;
}
}
// we have looped all the way without finding the value, so we can insert
if(!valueFound)
{
string strConn = Convert.ToString(ConfigurationManager.ConnectionStrings["connectionstring"]);
SqlConnection conn = new SqlConnection(strConn);
SqlCommand cmd = new SqlCommand(
"INSERT INTO tblVisitorcompany ([CompanyName], " +
"[Location1]) " +
"VALUES(#CompanyName, #Location1)", conn);
cmd.Parameters.AddWithValue("#Companyname", companyname);
cmd.Parameters.AddWithValue("#Location1", Location1);
conn.Open();
cmd.ExecuteNonQuery();
conn.Close();
}
Off course you could check if the value exists in a more efficient way, but this should at least solve your specific problem.
I have two application(A & B) and a SQL table (Microsoft SQL Server 2014) with columns as below:
|| Guid || Random || WriteTime || Flag || ReadTime ||
Application A will insert the new row with new Guid, random, WriteTime and Flag.
(Code A)
private void WriteIntoDB(int random)
{
SqlCommand cmd = new SqlCommand();
using (SqlConnection conn = new SqlConnection(connectionString))
{
conn.Open();
using (SqlTransaction trans = conn.BeginTransaction())
{
SqlCommand command = conn.CreateCommand();
command.Connection = conn;
command.Transaction = trans;
command.CommandText = string.Format("INSERT INTO " + "MyTable" + "([Guid],[Random],[WriteTime],[Flag])" + "VALUES(#Guid,#Random,#WriteTime,#Flag)");
command.Parameters.AddWithValue("#Guid", Guid.NewGuid());
command.Parameters.AddWithValue("#Random", random);
command.Parameters.AddWithValue("#WriteTime", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.fff"));
command.Parameters.AddWithValue("#Flag", 0);
command.ExecuteNonQuery();
trans.Commit();
}
}
This function will be called six times in TimerA_Tick(with TimerA interval = 1500 millisecond)
And Application B also has a TimerB with interval = 1000 millisecond. In TimerB_Tick, it will select all rows where Flag=0, and change it to 1, also fill in column 'ReadTime'.
(CODE B)
private DataSet DS = new DataSet();
private void ChangeFlag(SqlConnection conn)
{
string cmd = "SELECT * FROM MyTable WHERE Flag = 0";
SqlDataAdapter da = new SqlDataAdapter(cmd, conn);
DS.Tables["test"].Clear();
da.Fill(DS.Tables["test"]);
lock (DS.Tables["test"])
{
using (new SqlCommandBuilder(da))
{
DataRow[] rows = DS.Tables["test"].Select("Flag=0 ", "WriteTime ASC");
foreach (DataRow row in rows)
{
if (Convert.ToInt32(row["Random"]) <= 50)
{
//do something.
}
row["ReadTime"] = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.fff");
row["Flag"] = 1;
da.Update(DS.Tables["test"]);
}
}
}
}
I expect 'Random' which was written in the database first also to be processed by ApplicationB first. Most of the data was, but some of them were not...
The link below is part of that SQL Table, ordered with "WriteTime' ascendant.
There's a row that seems to be processed previously to the data which is written earlier. But I don't know why it happened(and more than one time), also got no idea how to fix it.
SqlCommand cmd = new SqlCommand();
cmd.Connection = con;
cmd.CommandType = CommandType.Text;
cmd = CreateParameterizedQuery();
SqlDataAdapter dap = new SqlDataAdapter();
dap.SelectCommand = cmd;
DataTable tbl = new DataTable();
dap.Fill(tbl);
if (tbl.Rows.Count > 0)
{
grid.DataSource = tbl;
}
The actual SQL Query will produce results in SQL Management Studio. However I am getting 0 Rows of Data. I set a breakpoint at tbl.Rows.Count and I see it's 0 and stepping will skip the necessary code to set the DataSource.
private SqlCommand CreateParameterizedQuery()
{
SqlCommand command = new SqlCommand();
string[] allTheseWords;
if (textBoxAllTheseWords.Text.Length > 0)
{
allTheseWords = textBoxAllTheseWords.Text.Split(' ');
string SQLQuery = "SELECT distinct [databaseName].[dbo].[customerTable].[name], [databaseName].[dbo].[customerTable].[dos], [databaseName].[dbo].[customerTable].[ACC], [databaseName].[dbo].[reportTable].[id], [databaseName].[dbo].[reportTable].[ACC], [databaseName].[dbo].[reportTable].[fullreport] FROM [databaseName].[dbo].[reportTable], [databaseName].[dbo].[customerTable] WHERE ";
int i = 0;
foreach (string word in allTheseWords)
{
var name = "#word" + (i++).ToString();
command.Parameters.AddWithValue(name, "'%" + word + "%'");
SQLQuery = SQLQuery + String.Format(" [databaseName].[dbo].[reportTable].[fullreport] LIKE {0} AND ", name);
}
SQLQuery = SQLQuery + " [databaseName].[dbo].[customerTable].[ACC] = [databaseName].[dbo].[reportTable].[ACC]";
command.CommandText = SQLQuery;
}
return command;
}
I am using WinForm with C# on Windows 8.
The SQLQuery variable contains this data when debugging
SELECT distinct [databaseName].[dbo].[customerTable].[name], [databaseName].[dbo].[customerTable].[dos], [databaseName].[dbo].[customerTable].[ACC], [databaseName].[dbo].[reportTable].[customerID], [databaseName].[dbo].[reportTable].[ACC], [databaseName].[dbo].[reportTable].[fullreport] FROM [databaseName].[dbo].[reportTable], [databaseName].[dbo].[customerTable] WHERE [databaseName].[dbo].[reportTable].[fullreport] LIKE #word0 AND [databaseName].[dbo].[customerTable].[ACC] = [databaseName].[dbo].[reportTable].[ACC]
debugMySQL is a method that spits out the SQL Query with the parameters substituted
public void debugMySQL()
{
string query = command.CommandText;
foreach (SqlParameter p in command.Parameters)
{
query = query.Replace(p.ParameterName, p.Value.ToString());
}
textBox1.Text = query;
}
The output looks like
SELECT distinct [databaseName].[dbo].[customerTable].[name], [databaseName].[dbo].[customerTable].[dos], [databaseName].[dbo].[customerTable].[ACC], [databaseName].[dbo].[reportTable].[id], [databaseName].[dbo].[reportTable].[ACC], [databaseName].[dbo].[reportTable].[fullreport] FROM [databaseName].[dbo].[reportTable], [databaseName].[dbo].[customerTable] WHERE [databaseName].[dbo].[reportTable].[fullreport] LIKE '%single%' AND [databaseName].[dbo].[customerTable].[ACC] = [databaseName].[dbo].[reportTable].[ACC]
You can see parametrized query with its argument value using SQL Profiler.
I am trying to insert a record and get its newly generated id by executing two queries one by one, but don't know why its giving me the following error.
Object cannot be cast from DBNull to other types
My code is as below: (I don't want to use sql stored procedures)
SqlParameter sqlParam;
int lastInsertedVideoId = 0;
using (SqlConnection Conn = new SqlConnection(ObjUtils._ConnString))
{
Conn.Open();
using (SqlCommand sqlCmd = Conn.CreateCommand())
{
string sqlInsertValues = "#Name,#Slug";
string sqlColumnNames = "[Name],[Slug]";
string sqlQuery = "INSERT INTO videos(" + sqlColumnNames + ") VALUES(" + sqlInsertValues + ");";
sqlCmd.CommandText = sqlQuery;
sqlCmd.CommandType = CommandType.Text;
sqlParam = sqlCmd.Parameters.Add("#Name", SqlDbType.VarChar);
sqlParam.Value = txtName.Text.Trim();
sqlParam = sqlCmd.Parameters.Add("#Slug", SqlDbType.VarChar);
sqlParam.Value = txtSlug.Text.Trim();
sqlCmd.ExecuteNonQuery();
//getting last inserted video id
sqlCmd.CommandText = "SELECT SCOPE_IDENTITY() AS [lastInsertedVideoId]";
using (SqlDataReader sqlDr = sqlCmd.ExecuteReader())
{
sqlDr.Read();
lastInsertedVideoId = Convert.ToInt32(sqlDr["lastInsertedVideoId"]);
}
}
}
//tags insertion into tag table
if (txtTags.Text.Trim().Length > 0 && lastInsertedVideoId > 0)
{
string sqlBulkTagInsert = "";
string[] tags = txtTags.Text.Split(new string[] { "," }, StringSplitOptions.RemoveEmptyEntries);
foreach (string tag in tags)
{
sqlBulkTagInsert += "INSERT INTO tags(VideoId, Tag) VALUES(" + lastInsertedVideoId + ", " + tag.Trim().ToLowerInvariant()+ "); ";
}
using (SqlConnection Conn = new SqlConnection(ObjUtils._ConnString))
{
Conn.Open();
using (SqlCommand sqlCmd = Conn.CreateCommand())
{
string sqlQuery = sqlBulkTagInsert;
sqlCmd.CommandText = sqlQuery;
sqlCmd.CommandType = CommandType.Text;
sqlCmd.ExecuteNonQuery();
}
}
}
And also if possible, please check is the above code coded well or we can optimize it more for improve performance?
Thanks
The call to SCOPE_IDENTITY() is not being treated as being in the same "scope" as the INSERT command that you're executing.
Essentially, what you need to do is change the line:
string sqlQuery = "INSERT INTO videos(" + sqlColumnNames + ") VALUES(" + sqlInsertValues + ");";
to:
string sqlQuery = "INSERT INTO videos(" + sqlColumnNames + ") VALUES(" + sqlInsertValues + "); SELECT SCOPE_IDENTITY() AS [lastInsertedVideoId]";
and then call
int lastVideoInsertedId = Convert.ToInt32(sqlCmd.ExecuteScalar());
instead of .ExecuteNonQuery and the code block following the "//getting last inserted video id" comment.
The SCOPE_IDENTITY() should be extracted from the first command (SELECT, RETURN or OUT) and passed into the next command. By that, I mean that the SELECT_IDENTITY() should be at the end of the first command. In SQL 2008 there is additional syntax for bring values back as part of the INSERT, which makes this simpler.
Or more efficiently: combine the commands into one to avoid round-trips.