I have a large CSV database of about 5MB with ZIP codes, cities, and states that I'm trying to import into a SQL Server CE database.
Using a single thread, the process is estimated to take about 3 hours to complete. While this is fine for getting the job done, I'd like to try and split up the task across multiple threads to cut down on the 3 hours total time. If I create a SqlCeConnection object on each thread, is it safe to run commands on each thread simultaneously?
I have a feeling that there would be issues with concurrency and deadlocks. Here is where I found the CSV database: http://www.unitedstateszipcodes.org/zip-code-database/
Here is my relevant code:
List<AddressSet> addressList;
public void OpenCSV(string file)
{
var addresses = from line in File.ReadAllLines(file).Skip(1)
let columns = line.Split(',')
select new AddressSet
{
ZipCode = columns[0].Replace("\"", "").Trim(),
City = columns[2].Replace("\"", "").Trim(),
State = columns[5].Replace("\"", "").Trim()
};
addressList = addresses.ToList();
Thread worker = new Thread(new ThreadStart(ProcessData));
worker.Start();
}
private void ProcessData()
{
try
{
int i = 1;
DateTime operationStart = DateTime.Now;
foreach (AddressSet address in addressList)
{
int stateId = InsertState(address.State);
int zipCodeId = InsertZipCode(address.ZipCode, stateId);
int cityId = InsertCity(address.City, stateId);
UpdateRelationships(zipCodeId, cityId);
float pct = i / (float)addressList.Count() * 100;
TimeSpan timeSinceStart = DateTime.Now.Subtract(operationStart);
TimeSpan totalTime = TimeSpan.FromMilliseconds(timeSinceStart.TotalMilliseconds / (pct/100));
TimeSpan timeLeft = totalTime - timeSinceStart;
//richTextBox1.BeginInvoke((MethodInvoker)(() => richTextBox1.Text = pct.ToString("N2") + "% (" + i + " of " + addressList.Count().ToString() + ") " + address.City + ", " + address.State + " " + address.ZipCode
// + "\nEstimated Total Time: " + totalTime.Days.ToString() + " days, " + totalTime.Hours.ToString() + " hours, " + totalTime.Minutes.ToString() + " minutes" +
// " - Time Left: " + timeLeft.Days.ToString() + " days, " + timeLeft.Hours.ToString() + " hours, " + timeLeft.Minutes.ToString() + " minutes"));
richTextBox1.BeginInvoke((MethodInvoker)(() => richTextBox1.Text = pct.ToString("N2") + "% (" + i + " of " + addressList.Count().ToString() + ") " + address.City + ", " + address.State + " " + address.ZipCode
+ "\nEstimated Total Time: " + totalTime.ToString("h'h 'm'm 's's'") +
"\nTime Left: " + timeLeft.ToString("h'h 'm'm 's's'") +
"\nRunning Time: " + timeSinceStart.ToString("h'h 'm'm 's's'")));
richTextBox1.BeginInvoke((MethodInvoker)(() => richTextBox1.SelectionStart = richTextBox1.Text.Length));
richTextBox1.BeginInvoke((MethodInvoker)(() => richTextBox1.ScrollToCaret()));
i++;
}
this.Invoke(new Action(() =>
{
MessageBox.Show("Done!");
btnChooseCSV.Enabled = true;
}));
}
catch (Exception ex)
{
this.Invoke(new Action(() =>
{
MessageBox.Show(ex.Message);
}));
}
}
private int InsertZipCode(string zipCode, int stateId)
{
string connstr = System.Configuration.ConfigurationManager.ConnectionStrings["AddressInformation"].ConnectionString;
SqlCeConnection connection = new SqlCeConnection(connstr);
connection.Open();
SqlCeCommand command = new SqlCeCommand("SELECT COUNT(*) FROM ZipCode WHERE ZipCode = #ZipCode", connection);
command.Parameters.AddWithValue("ZipCode", zipCode);
int result = (int)command.ExecuteScalar();
// if nothing found, insert
if (result == 0)
{
command = new SqlCeCommand("INSERT INTO ZipCode(ZipCode, StateId) VALUES(#ZipCode, #StateId)", connection);
command.Parameters.AddWithValue("ZipCode", zipCode);
command.Parameters.AddWithValue("StateId", stateId);
command.ExecuteNonQuery();
command = new SqlCeCommand("SELECT ##IDENTITY", connection);
}
if (result == 1)
{
command = new SqlCeCommand("SELECT ZipCodeId FROM ZipCode WHERE ZipCode = #ZipCode", connection);
command.Parameters.AddWithValue("ZipCode", zipCode);
}
string test = command.ExecuteScalar().ToString();
result = int.Parse(test);
connection.Close();
return result;
}
private int InsertCity(string city, int stateId)
{
string connstr = System.Configuration.ConfigurationManager.ConnectionStrings["AddressInformation"].ConnectionString;
SqlCeConnection connection = new SqlCeConnection(connstr);
connection.Open();
SqlCeCommand command = new SqlCeCommand("SELECT COUNT(*) FROM City WHERE CityName = #City", connection);
command.Parameters.AddWithValue("City", city);
int result = (int)command.ExecuteScalar();
// if nothing found, insert
if (result == 0)
{
command = new SqlCeCommand("INSERT INTO City(CityName, StateId) VALUES(#City, #StateId)", connection);
command.Parameters.AddWithValue("City", city);
command.Parameters.AddWithValue("StateId", stateId);
command.ExecuteNonQuery();
command = new SqlCeCommand("SELECT ##IDENTITY", connection);
}
if (result == 1)
{
command = new SqlCeCommand("SELECT CityId FROM City WHERE CityName = #City", connection);
command.Parameters.AddWithValue("City", city);
}
string test = command.ExecuteScalar().ToString();
result = int.Parse(test);
connection.Close();
return result;
}
private int InsertState(string state)
{
string connstr = System.Configuration.ConfigurationManager.ConnectionStrings["AddressInformation"].ConnectionString;
SqlCeConnection connection = new SqlCeConnection(connstr);
connection.Open();
SqlCeCommand command = new SqlCeCommand("SELECT COUNT(*) FROM State WHERE State = #State", connection);
command.Parameters.AddWithValue("State", state);
int result = (int)command.ExecuteScalar();
// if nothing found, insert
if (result == 0)
{
command = new SqlCeCommand("INSERT INTO State(State) VALUES(#State)", connection);
command.Parameters.AddWithValue("State", state);
command.ExecuteNonQuery();
command = new SqlCeCommand("SELECT ##IDENTITY", connection);
}
if (result == 1)
{
command = new SqlCeCommand("SELECT StateId FROM State WHERE State = #State", connection);
command.Parameters.AddWithValue("State", state);
}
string test = command.ExecuteScalar().ToString();
result = int.Parse(test);
connection.Close();
return result;
}
private void UpdateRelationships(int zipCodeId, int cityId)
{
string connstr = System.Configuration.ConfigurationManager.ConnectionStrings["AddressInformation"].ConnectionString;
SqlCeConnection connection = new SqlCeConnection(connstr);
connection.Open();
SqlCeCommand command = new SqlCeCommand("INSERT INTO CityZipCode(CityId, ZipCodeId) VALUES(#CityId, #ZipCodeId)", connection);
command.Parameters.AddWithValue("CityId", cityId);
command.Parameters.AddWithValue("ZipCodeId", zipCodeId);
command.ExecuteNonQuery();
connection.Close();
}
Edit:
Just to clarify, I'm not just simply inserting each row of information from the CSV file. I'm changing how the data is laid out by inserting each respective item into separate tables and adding relationships between each entity.
For example, a city can have multiple zip codes and a zip code can sometimes cover multiple cities so that would be represented by a many to many relationship. Cities and zip codes have only one state so that relationship is many to one.
I have a table for cities, zip codes, and states. I also have a table for relating cities to zip codes. I will need to modify my relationship table schema to take into effect that cities with the same name may exist in multiple states. The relationship table should really be a set including the city, state and zip code and not just the city and zip code.
My end goal is to distribute the SQL Server CE database with password protection with another application for city, state and zip code validation. I don't want to distribute the CSV database as anyone could change that to pass validation.
You must create a connection object per thread, it is not safe for multithreading:
SqlCeConnection Class
Edited
SQL CE objects are not thread-safe and are not thread affinitive
either. If an instance of SqlCeConnection or SqlCeTransaction is
shared across threads without ensuring thread safety, then that may
lead to Access Violation exception.
It is recommended that each thread should use a separate connection
than sharing across. If there is really a need for sharing SQL CE
objects across threads, then the application should serialize access
to these objects.
Multithreaded programming with SQL Server Compact
Why you don't use SQL Server Compact Toolbox You can use it, which generates INSERT statements based on a CSV file.
or use Conversion of CSV to SQLCE database app
Just a suggestion, I am doing the same kind of thing, and this is what I have, it is extremely fast compared to the simple solution
public static DataTable CSVToDataTable(string path, string name)
{
return CSVToDataTable(Path.Combine(path, name));
}
public static DataTable CSVToDataTable(string path)
{
DataTable res = new DataTable();
if (!File.Exists(path))
{
return res;
}
using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
using (StreamReader re = new StreamReader(stream))
{
if (re.EndOfStream)
return res;
string line = re.ReadLine();
if (line.IsNullOrWhiteSpace())
return res;
string[] headers = LineToArray(line);
foreach (string header in headers)
{
res.Columns.Add(header);
}
int i = 0;
string[] cells = new string[0];
DataRow row = null;
while (!re.EndOfStream)
{
line = re.ReadLine();
if (line.IsNullOrWhiteSpace())
continue;
cells = LineToArray(line);
row = res.NewRow();
for (i = 0; i < headers.Length && i < cells.Length; i += 1)
{
row[i] = cells[i];
}
res.Rows.Add(row);
}
}
}
return res;
}
private static string[] LineToArray(string line, char delimiter = ',')
{
if (line.Contains("\""))
{
List<string> l = new List<string>();
bool inq = false;
string cell = string.Empty;
char lastCh = 'x';
foreach (char ch in line)
{
if (ch == '"')
{
if (cell.Length == 0)
{
inq = true;
}
else if (lastCh == '\\')
{
cell += ch;
}
else
{
inq = false;
}
}
else if (delimiter == ch)
{
if (inq)
{
cell += ch;
}
else
{
l.Add(cell);
inq = false;
cell = string.Empty;
}
}
else
{
cell += ch;
}
if (inq)
lastCh = ch;
else
lastCh = 'x';
}
return l.ToArray();
}
else
{
return line.Split(new String[] { delimiter.ToString() }, StringSplitOptions.None);
}
}
public void insert(string path, string name, string table, bool KeepNulls){
DataTable data = CSVToDataTable(path, name);
//do data manipulation here
SqlCeBulkCopyOptions options = new SqlCeBulkCopyOptions();
if (KeepNulls)
{
options = options |= SqlCeBulkCopyOptions.KeepNulls;
}
using (SqlCeBulkCopy bc = new SqlCeBulkCopy(Fastway_Remote_Agent.Properties.Settings.Default.DatabaseConnectionString, options))
{
bc.DestinationTableName = table;
bc.WriteToServer(data);
}
}
Using this library: http://sqlcebulkcopy.codeplex.com/
Also for thread pooling (Change it to meet your needs):
/// <summary>
/// Manages open connections on a per-thread basis
/// </summary>
public abstract class SqlCeConnectionPool
{
private static Dictionary<int, DBCon> threadConnectionMap = new Dictionary<int, DBCon>();
private static Dictionary<int, Thread> threadMap = new Dictionary<int, Thread>();
/// <summary>
/// The connection map
/// </summary>
public static Dictionary<int, DBCon> ThreadConnectionMap
{
get { return SqlCeConnectionPool.threadConnectionMap; }
}
/// <summary>
/// Gets the connection string.
/// </summary>
/// <value>The connection string.</value>
public static ConnectionString ConnectionString
{
get { return global::ConnectionString.Default; }
}
/// <summary>
/// Gets a connection for this thread, maintains one open one of each.
/// </summary>
/// <remarks>Don't do this with anything but SQL compact edition or you'll run out of connections - compact edition is not
/// connection pooling friendly and unloads itself too often otherwise so that is why this class exists</remarks>
/// <returns>An open connection</returns>
public static DBCon Connection
{
get
{
lock (threadConnectionMap)
{
//do some quick maintenance on existing connections (closing those that have no thread)
List<int> removeItems = new List<int>();
foreach (var kvp in threadConnectionMap)
{
if (threadMap.ContainsKey(kvp.Key))
{
if (!threadMap[kvp.Key].IsAlive)
{
//close the connection
if (!kvp.Value.Disposed)
kvp.Value.Dispose();
removeItems.Add(kvp.Key);
}
}
else
{
if (!kvp.Value.Disposed)
kvp.Value.Dispose();
removeItems.Add(kvp.Key);
}
}
foreach (int i in removeItems)
{
threadMap.Remove(i);
threadConnectionMap.Remove(i);
}
//now issue the appropriate connection for our current thread
int threadId = Thread.CurrentThread.ManagedThreadId;
DBCon connection = null;
if (threadConnectionMap.ContainsKey(threadId))
{
connection = threadConnectionMap[threadId];
if (connection.Disposed)
{
if (threadConnectionMap.ContainsKey(threadId))
threadConnectionMap.Remove(threadId);
if (threadMap.ContainsKey(threadId))
threadMap.Remove(threadId);
connection = null;
}
else if (connection.Connection.State == ConnectionState.Broken)
{
connection.Dispose();
if (threadConnectionMap.ContainsKey(threadId))
threadConnectionMap.Remove(threadId);
if (threadMap.ContainsKey(threadId))
threadMap.Remove(threadId);
connection = null;
}
else if (connection.Connection.State == ConnectionState.Closed)
{
connection.Dispose();
if (threadConnectionMap.ContainsKey(threadId))
threadConnectionMap.Remove(threadId);
if (threadMap.ContainsKey(threadId))
threadMap.Remove(threadId);
connection = null;
}
}
if (connection == null)
{
connection = new DBCon(ConnectionString);
//connection.Connection.Open();
if (threadConnectionMap.ContainsKey(threadId))
threadConnectionMap[threadId] = connection;
else
threadConnectionMap.Add(threadId, connection);
if (threadMap.ContainsKey(threadId))
threadMap[threadId] = Thread.CurrentThread;
else
threadMap.Add(threadId, Thread.CurrentThread);
}
return connection;
}
}
}
}
Related
I am working on software that make changes in database through GUI. I want to compact database after user clicks save. After save user can continue to use software or close, so I am not using "using". I have created databaseAccess object which holds OleDbConnection connection object with few others. This is my database access class.
using System;
using System.Collections.Generic;
using System.Data.OleDb;
using System.Data;
using System.Diagnostics;
using System.Windows.Forms;
namespace TreeTool
{
public class DataBaseAccess
{
#region Properties
private string m_directory;
public List<string> selectedTableNames;
private Dictionary<String, DataTable> selectedTables;
private OleDbConnection mdbConnection;
DataTable dataTable;
//Constructor
public DataBaseAccess()
{
selectedTableNames = new List<string>();
selectedTables = new Dictionary<string, DataTable>();
}
public string directory
{
get
{
return m_directory;
}
set
{
m_directory = value;
}
}
#endregion
public List<string> GetAllTableNames()
{
if (dataTable != null)
{
List<string> tableList = new List<string>();
for (int i = 0; i < dataTable.Rows.Count; i++)
{
string TableName = dataTable.Rows[i][2].ToString();
tableList.Add(TableName);
}
return tableList;
}
return null;
}
/// <summary>
/// Returns Table Columns
/// </summary>
/// <returns></returns>
public DataTable GetTable(string TableName)
{
DataTable mdbTable;
if (selectedTables.TryGetValue(TableName, out mdbTable))
{
return mdbTable;
}
else
{
mdbTable = new DataTable();
//mdbConnection.Open();
string mdbCommandString = "SELECT * FROM [" + TableName + "]";
OleDbDataAdapter QueryCommand = new OleDbDataAdapter(mdbCommandString, mdbConnection);
QueryCommand.Fill(mdbTable);
//mdbConnection.Close();
selectedTables.Add(TableName, mdbTable);
return mdbTable;
}
}
public void SetTable(String TableName, DataTable dataTable)
{
//mdbConnection.Open();
OleDbCommand ac = new OleDbCommand("delete from [" + TableName + "]", mdbConnection);
ac.ExecuteNonQuery();
foreach (DataRow row in dataTable.Rows)
{
String query = "INSERT INTO [" + TableName + "] (TaskID, HTMLTopic, nRelative, [Group], nKey,"
+ " [nText], nImage, nSelImage, nFontName, nFontInfo, Keywords) VALUES (#TaskID,"
+ " #HTMLTopic, #nRelative, #Group, #nKey, #nText, #nImage, #nSelImage, #nFontName, "
+ " #nFontInfo, #Keywords)";
OleDbCommand command = new OleDbCommand(query, mdbConnection);
command.Parameters.AddWithValue("#TaskID", row["TaskID"]);
command.Parameters.AddWithValue("#HTMLTopic", row["HTMLTopic"]);
command.Parameters.AddWithValue("#nRelative", row["nRelative"]);
command.Parameters.AddWithValue("#Group", row["Group"]);
command.Parameters.AddWithValue("#nKey", row["nKey"]);
command.Parameters.AddWithValue("#nText", row["nText"]);
command.Parameters.AddWithValue("#nImage", row["nImage"]);
command.Parameters.AddWithValue("#nSelImage", row["nSelImage"]);
command.Parameters.AddWithValue("#nFontName", row["nFontName"]);
command.Parameters.AddWithValue("#nFontInfo", row["nFontInfo"]);
command.Parameters.AddWithValue("#Keywords", row["Keywords"]);
command.ExecuteNonQuery();
}
//mdbConnection.Close();
}
internal bool validTable(string TableName)
{
DataTable mdbTable = new DataTable();
//mdbConnection.Open();
string mdbCommandString = "SELECT * FROM [" + TableName + "]";
OleDbDataAdapter QueryCommand = new OleDbDataAdapter(mdbCommandString, mdbConnection);
QueryCommand.Fill(mdbTable);
//mdbConnection.Close();
// check if table contains all columns necessary
String[] columnNames = new string[] { "TaskID" , "HTMLTopic", "nRelative", "Group", "nKey",
"nText", "nImage", "nSelImage", "nFontName", "nFontInfo", "Keywords"};
Boolean missingColumn = false;
DataColumnCollection columns = mdbTable.Columns;
foreach (String columnName in columnNames)
{
if (columns.Contains(columnName) == false)
{
// print the message
MessageBox.Show("Database: " + directory + " Table: " + TableName + " is missing column \"" + columnName
+ "\". Add it to make changes.",
"Missing column",
MessageBoxButtons.OK,
MessageBoxIcon.Exclamation,
MessageBoxDefaultButton.Button1);
missingColumn = true;
}
}
if (missingColumn == true)
{
return false;
}
return true;
}
public void insertTable(String tableName)
{
selectedTableNames.Add(tableName);
}
public List<String> getSelectedTables()
{
return selectedTableNames;
}
public Boolean isConnected()
{
if (mdbConnection == null)
{
return false;
}
return true;
}
public void connect()
{
if (mdbConnection == null)
{
String m_mdbDirectory = #"Provider=Microsoft.JET.OLEDB.4.0;Data Source=" + m_directory;
mdbConnection = new OleDbConnection(m_mdbDirectory);
mdbConnection.Open();
string[] restrictions = new string[4];
restrictions[3] = "Table";
dataTable = mdbConnection.GetSchema("TABLES", restrictions);
//mdbConnection.Close();
}
}
public void disconnect()
{
mdbConnection.Close();
mdbConnection.Dispose();
GC.SuppressFinalize(mdbConnection);
mdbConnection = null;
}
public void clearSelectedTables()
{
selectedTableNames.Clear();
}
}
}
Save and compact functions are like this
private void save()
{
foreach(DataBaseAccess database in databases)
{
// save changes code
database.disconnect();
CompactAndRepairAccessDB(database.directory);
database.connect();
}
}
private void CompactAndRepairAccessDB(string accessFile)
{
string tempFile = #"temp.mdb";
FileInfo temp = new FileInfo(tempFile);
// Required COM reference for project:
// Microsoft Office 14.0 Access Database Engine Object Library
var dbe = new Microsoft.Office.Interop.Access.Dao.DBEngine();
try
{
dbe.CompactDatabase(accessFile, tempFile);
temp.CopyTo(accessFile, true);
temp.Delete();
}
catch (Exception e)
{
Console.WriteLine("Error: " + e.Message);
}
}
The exception happens on line "dbe.CompactDatabase(accessFile, tempFile);".
In the code in all the methods where you use OleDbDataAdapter and OleDbCommand make sure to use the using-Pattern on these objects. They can keep the mdb file open even though the actual connection is already disposed.
Methods that require modification seem to be GetTable, SetTable and ValidTable.
I'm working on an autocomplete script.
When user stop typing, his input is send to the server which will look for some matches in DB.
Eg: If looking for people "Obam" should return "Barack Obama".
I want this search to be limited to ~500ms. If it takes longer, I want to abort the search and only return results found in time.
I start by looking for a perfect match (this one won't be interrupted), then I am looking for a partial match (this one can be interrupted) :
private static MySqlConnection conn = new MySqlConnection(ConfigurationManager.AppSettings["CxMySql"].ToString());
protected void Page_Load(object sender, EventArgs e)
{
conn.Open();
SearchTerm(table,term,domaine);
conn.Close(); // TimeoutException HERE
conn.Dispose();
}
private void SearchTerm(string table,string term,string domaine)
{
Dictionary<int, Scoring> results = new Dictionary<int, Scoring>();
var requete = "SELECT m.id, m.nom as label FROM marque m WHERE m.nom = '" + term + "'";
var Cmd = new MySqlCommand(requete, conn);
using (var Rd = Cmd.ExecuteReader())
{
while (Rd.Read())
{
results.Add(int.Parse(Rd["id"].ToString()), new Scoring()
{
score = 1000,
value = Rd["label"].ToString()
});
}
}
// Here it should be 500, but I put 1 to force troubles to appear.
RunWithTimeout(() => FindOtherBrands(term, ref results), TimeSpan.FromMilliseconds(1));
var resultsList = results.ToList();
resultsList.Sort(
delegate(KeyValuePair<int, Scoring> firstPair,
KeyValuePair<int, Scoring> nextPair)
{
return nextPair.Value.score - firstPair.Value.score;
}
);
}
private void FindOtherBrands(string term, ref Dictionary<int, Scoring> results)
{
MySqlCommand Cmd;
string requete;
requete = "SELECT m.id, m.nom as label FROM marque m WHERE m.nom LIKE '" + term + "%'";
Cmd = new MySqlCommand(requete, conn);
var Rd = Cmd.ExecuteReader(); // NullReferenceException HERE
while (Rd != null && Rd.Read())
{
int id = int.Parse(Rd["id"].ToString());
if (!results.ContainsKey(id))
{
results.Add(id, new Scoring()
{
score = 100,
value = Rd["label"].ToString()
});
}
}
Rd.Close();
requete = "SELECT m.id, m.nom as label FROM marque m WHERE m.nom LIKE '%" + term + "%'";
Cmd = new MySqlCommand(requete, conn);
Rd = Cmd.ExecuteReader();
while (Rd != null && Rd.Read())
{
int id = int.Parse(Rd["id"].ToString());
if (!results.ContainsKey(id))
{
results.Add(id, new Scoring()
{
score = 10,
value = Rd["label"].ToString()
});
}
}
Rd.Close();
}
I found the RunWithTimeout method here : stop executing code in thread after 30s
bool RunWithTimeout(ThreadStart threadStart, TimeSpan timeout)
{
Thread workerThread = new Thread(threadStart);
workerThread.Start();
bool finished = true;
if (!workerThread.Join(timeout))
{
workerThread.Abort();
finished = false;
}
return finished;
}
Scoring is a struct for an easy sorting of results
private struct Scoring
{
public string value;
public int score;
}
The aim is to have results (not necessarily all) fast.
PROBLEMS
I randomly get Timeoutexception after ~30s on the conn.Close(); line.
I randomly get NullReferenceException on the first Cmd.ExecuteReader(); call in FindOtherBrands.
Can anyone explain me why ?
Am I doing something wrong or is there a workaround ?
I guess the TimeoutException is because I am trying to close connection during command execution, can I drop/cancel that query ?
I would take a different approach. Since you're querying a database, which is naturally asynchronous, you can use async-await for querying the data. With that, you can pass a CancellationToken which is set to a timeout, which you'll monitor with every read:
For example:
private async Task FindOtherBrands(string term,
Dictionary<int, Scoring> results,
CancellationToken cancellationToken)
{
MySqlCommand cmd;
string requete;
requete = "SELECT m.id, m.nom as label
FROM marque m
WHERE m.nom LIKE '" + term + "%'";
cmd = new MySqlCommand(requete, conn);
var Rd = await cmd.ExecuteReaderAsync();
while (Rd != null && await Rd.ReadAsync())
{
cancellationToken.ThrowIfCancellationRequested();
int id = int.Parse(Rd["id"].ToString());
if (!results.ContainsKey(id))
{
results.Add(id, new Scoring()
{
score = 100,
value = Rd["label"].ToString()
});
}
}
Rd.Close();
requete = "SELECT m.id, m.nom as label
FROM marque m
WHERE m.nom LIKE '%" + term + "%'";
cmd = new MySqlCommand(requete, conn);
Rd = await Cmd.ExecuteReaderAsync();
while (Rd != null && await Rd.ReadAsync())
{
cancellationToken.ThrowIfCancellationRequest();
int id = int.Parse(Rd["id"].ToString());
if (!results.ContainsKey(id))
{
results.Add(id, new Scoring()
{
score = 10,
value = Rd["label"].ToString()
});
}
}
Rd.Close();
}
And when you invoke it, all you need is to wrap it in a try-catch and pass a CancellationToken:
private async Task<bool> RunWithTimeoutAsync(TimeSpan timeout)
{
bool finished;
try
{
var cancellationTokenSource = new CancellationTokenSource(timeout);
await FindOtherBrandsAsnyc(term,
results,
cancellationTokenSource.CancellationToken);
finished = true;
}
catch (OperationCanceledException e)
{
// Handle
}
return finished;
}
Side note - Your query is prone to SQL Injection. You shouldn't used string concatenation. Use query parameters instead.
I have been struggling to get the right c# code for getting the values after a PRAGMA table_info query.
Since my edit with extra code was rejected in this post, I made this question for other people that would otherwise waste hours for a fast solution.
Assuming you want a DataTable with the list of field of your table:
using (var con = new SQLiteConnection(preparedConnectionString))
{
using (var cmd = new SQLiteCommand("PRAGMA table_info(" + tableName + ");"))
{
var table = new DataTable();
cmd.Connection = con;
cmd.Connection.Open();
SQLiteDataAdapter adp = null;
try
{
adp = new SQLiteDataAdapter(cmd);
adp.Fill(table);
con.Close();
return table;
}
catch (Exception ex)
{ }
}
}
Return result is:
cid: id of the column
name: the name of the column
type: the type of the column
notnull: 0 or 1 if the column can contains null values
dflt_value: the default value
pk: 0 or 1 if the column partecipate to the primary key
If you want only the column names into a List you can use (you have to include System.Data.DataSetExtension):
return table.AsEnumerable().Select(r=>r["name"].ToString()).ToList();
EDIT: Or you can avoid the DataSetExtension reference using this code:
using (var con = new SQLiteConnection(preparedConnectionString))
{
using (var cmd = new SQLiteCommand("PRAGMA table_info(" + tableName + ");"))
{
var table = new DataTable();
cmd.Connection = con;
cmd.Connection.Open();
SQLiteDataAdapter adp = null;
try
{
adp = new SQLiteDataAdapter(cmd);
adp.Fill(table);
con.Close();
var res = new List<string>();
for(int i = 0;i<table.Rows.Count;i++)
res.Add(table.Rows[i]["name"].ToString());
return res;
}
catch (Exception ex){ }
}
}
return new List<string>();
There are a lot of PRAGMA statements that you can use in SQLite, have a look at the link.
About the using statement: it's very simple, it is used to be sure that disposable objects will be disposed whatever can happen in your code: see this link or this reference
Code:
DB = new SQLiteConnection(#"Data Source="+DBFileName);
DB.Open();
SQLiteCommand command = new SQLiteCommand("PRAGMA table_info('tracks')", DB);
DataTable dataTable = new DataTable();
SQLiteDataAdapter dataAdapter = new SQLiteDataAdapter(command);
dataAdapter.Fill(dataTable);
DB.Close();
foreach (DataRow row in dataTable.Rows) {
DBColumnNames.Add((string)row[dataTable.Columns[1]]); }
//Out(String.Join(",",
DBColumnNames.ToArray()));//debug
All elements in the resulted rows:
int cid, string name, string type,int notnull, string dflt_value, int pk
More info on PRAGMA
Not sure if this exactly what you are after but this is how I have grabbed the data and subsequently used it. Hope it helps!
Obviously the switch is not covering all eventualities, just those I have needed to so far.
/// <summary>
/// Allows the programmer to easily update rows in the DB.
/// </summary>
/// <param name="tableName">The table to update.</param>
/// <param name="data">A dictionary containing Column names and their new values.</param>
/// <param name="where">The where clause for the update statement.</param>
/// <returns>A boolean true or false to signify success or failure.</returns>
public bool Update(String tableName, Dictionary<String, String> data, String where)
{
String vals = "";
Boolean returnCode = true;
//Need to determine the dataype of fields to update as this affects the way the sql needs to be formatted
String colQuery = "PRAGMA table_info(" + tableName + ")";
DataTable colDataTypes = GetDataTable(colQuery);
if (data.Count >= 1)
{
foreach (KeyValuePair<String, String> pair in data)
{
DataRow[] colDataTypeRow = colDataTypes.Select("name = '" + pair.Key.ToString() + "'");
String colDataType="";
if (pair.Key.ToString()== "rowid" || pair.Key.ToString()== "_rowid_" || pair.Key.ToString()=="oid")
{
colDataType = "INT";
}
else
{
colDataType = colDataTypeRow[0]["type"].ToString();
}
colDataType = colDataType.Split(' ').FirstOrDefault();
if ( colDataType == "VARCHAR")
{
colDataType = "VARCHAR";
}
switch(colDataType)
{
case "INTEGER": case "INT": case "NUMERIC": case "REAL":
vals += String.Format(" {0} = {1},", pair.Key.ToString(), pair.Value.ToString());
break;
case "TEXT": case "VARCHAR": case "DATE": case "DATETIME":
vals += String.Format(" {0} = '{1}',", pair.Key.ToString(), pair.Value.ToString());
break;
}
}
vals = vals.Substring(0, vals.Length - 1);
}
try
{
string sql = String.Format("update {0} set {1} where {2};", tableName, vals, where);
//dbl.AppendLine(sql);
dbl.AppendLine(sql);
this.ExecuteNonQuery(sql);
}
catch(Exception crap)
{
OutCrap(crap);
returnCode = false;
}
return returnCode;
}
I am trying to migrate data from Oracle to SQL.
I already created Table name and field Name.
Type and Size are same on both: e.g.
at Oracle Varchar2(11), On SQL VARCHAR(11)
at Oracle Date, On SQL datetime2(0).
In my application I'm using SqlBulkCopy function. But I face the above error on Date filed.
private void Processing(string sPath)
{
string AppPath = Application.StartupPath.ToString();
IniFile myIni = new IniFile(sPath);
string allTable = myIni.IniReadValue("TABLENAME", "TABLE");
string[] alltables = allTable.Split(',');
foreach (var tableName in alltables)
{
string whereSqlQuery = string.Empty;
string sOrderBySqlQuery = string.Empty;
string sSQLSelect = string.Empty;
string sRCount = string.Empty;
whereSqlQuery = myIni.IniReadValue(tableName, "WHERE");
TableName = tableName;
sOrderBySqlQuery = myIni.IniReadValue(tableName, "ORDERBY");
sSQLSelect = myIni.IniReadValue(tableName, "SQLSELECT");
//sRCount = myIni.IniReadValue(tableName, "RCOUNT");
if (radAuto.Checked == true)
ConnectAndQuery(tableName, whereSqlQuery, sOrderBySqlQuery, sSQLSelect, sRCount);
else
ConnectAndQuery(tableName, whereSqlQuery, sOrderBySqlQuery);
}
}
private void ConnectAndQuery(string strTableName, string strWhere, string strOrderBy, string sSqlSelect, string sRcount)
{
Cursor.Current = Cursors.WaitCursor;
string connectionString = GetConnectionString();
// Using
OracleConnection connection = new OracleConnection();
SqlConnection oConn = default(SqlConnection);
DataTable dtSQL = null;
DateTime dtLog = DateTime.Now;
try
{
//Opening Oracle DB Connection
connection.ConnectionString = connectionString;
connection.Open();
ShownLog(string.Format("Table = {0} , -- STARTING --", strTableName));
ShownLog("Oracle Connection Opened, OK");
OracleCommand command = connection.CreateCommand();
if (!string.IsNullOrEmpty(strWhere))
SqlQuery = "SELECT * FROM " + strTableName + " WHERE "+ strWhere;
else
SqlQuery = string.Format("SELECT * FROM {0} ", strTableName);
if (!string.IsNullOrEmpty(strOrderBy))
SqlQuery += " ORDER BY " + strOrderBy;
command.CommandText = SqlQuery;
System.DateTime startTime = System.DateTime.Now;
ShownLog("Starting Date Time : " + startTime);
DataTable dtTotalInsertCount = new DataTable(strTableName);
if (!string.IsNullOrEmpty(sRcount))
{
//OracleDataAdapter adpCount = new OracleDataAdapter(sRcount, connectionString);
////adpCount.FillSchema(dtTotalInsertCount, SchemaType.Source);
//adpCount.Fill(dtTotalInsertCount);
}
OracleDataAdapter adapter = new OracleDataAdapter(SqlQuery, connectionString);
DataTable dt = new DataTable(strTableName);
adapter.FillSchema(dt, SchemaType.Source);
bool valid = true;
bool bCheck = true;
int count = 1000;
int start = 0;
string sLogQuery = SqlQuery.Replace("'", "");
dtLog = DateTime.Now;
//Insert to SQL Log Table
Insert(strTableName, dtLog, sLogQuery);
int iLogCount = 0;
int iActualCount = 0;
do
{
int iEnd = 0;
adapter.Fill(start, count, dt);
valid = dt.Rows.Count > 0 ? true : false;
if (valid)
{
iLogCount += dt.Rows.Count;
iEnd = start + dt.Rows.Count;
ShownLog("No of data Rows retrieved from Oracle, Count = " + dt.Rows.Count);
//Create the SQL Server Table
oConn = new SqlConnection(txtDestinationConnString.Text);
oConn.Open();
ShownLog("SQL Connection Opened, OK");
if (string.IsNullOrEmpty(sSqlSelect))
bCheck = false;
if (bCheck)
{
ShownLog("Data Comparision Start : " + System.DateTime.Now);
//If SQL Select Statement has
if (!string.IsNullOrEmpty(sSqlSelect))
{
SqlCommand cmd = new SqlCommand();
cmd.CommandText = sSqlSelect;
cmd.Connection = oConn;
cmd.CommandType = CommandType.Text;
SqlDataReader reader = cmd.ExecuteReader();
dtSQL = new DataTable();
dtSQL.Load(reader);
cmd = null;
reader = null;
//Return record has more than 0
ShownLog("Start Duplicate Checking : " + System.DateTime.Now);
if (dtSQL.Rows.Count > 0)
{
foreach (DataRow dr in dtSQL.Rows)
{
DataRow[] result;
if (strTableName == "ITK_STAFF" || strTableName == "ITK_FLIGHT")
{
result = dt.Select("URNO='" + dr[0].ToString() + "'");
}
else
result = dt.Select("URNO=" + dr[0]);
if (result.Length > 0)
{
foreach (var drRemove in result)
{
dt.Rows.Remove(drRemove);
}
}
}
}
ShownLog("End Duplicate Checking : " + System.DateTime.Now);
}
ShownLog("Actual No.s of records to insert : " + dt.Rows.Count);
if (dt.Rows.Count == count)
bCheck = false;
ShownLog("Data Comparision End : " + System.DateTime.Now);
}
if (valid)
{
ShownLog(System.DateTime.Now + " Copying " + strTableName + " From : " + start.ToString() + " To : " + iEnd);
}
start += count;
if (dt.Rows.Count > 0)
{
iActualCount += dt.Rows.Count;
CopyData(dt, oConn);
dt.Rows.Clear();
}
oConn.Close();
oConn = null;
}
} while (valid);
System.DateTime endTime = System.DateTime.Now;
ShownLog("Ending DateTime: " + endTime);
//msgOut("No of rows copied from oracle to SQL Server , Count = " + dt.Rows.Count);
TimeSpan diffTime = endTime.Subtract(startTime);
ShownLog(String.Format("Time Difference is Days : {0}, Hours : {1}, Minites : {2}, seconds : {3} ,Milliseconds : {4}", diffTime.Days, diffTime.Hours, diffTime.Minutes, diffTime.Seconds, diffTime.Milliseconds));
ShownLog(string.Format("Table = {0} , -- FINISHED --", strTableName));
ShownLog(string.Empty);
Update(iLogCount, iActualCount, strTableName, dtLog, sLogQuery, " ", true);
Cursor.Current = Cursors.Default;
}
catch (Exception ex)
{
Cursor.Current = Cursors.Default;
ShownLog(ex.ToString());
ShownLog(string.Empty);
Update(0, 0, strTableName, dtLog, "", ex.ToString(), false);
((IDisposable)connection).Dispose();
if (oConn != null)
oConn.Close();
RadioButtonControl();
}
finally
{
Cursor.Current = Cursors.Default;
((IDisposable)connection).Dispose();
if (oConn != null)
oConn.Close();
RadioButtonControl();
}
}
private void CopyData(DataTable sourceTable, SqlConnection destConnection)
{
// Using
SqlBulkCopy s = new SqlBulkCopy(destConnection);
try
{
s.DestinationTableName = TableName;
s.NotifyAfter = 10000;
s.SqlRowsCopied += new SqlRowsCopiedEventHandler(s_SqlRowsCopied);
s.WriteToServer(sourceTable);
s.Close();
}
finally
{
((IDisposable)s).Dispose();
}
}
I have written a custom class to handle database queries to a remote and local MySQL database, however when I do a nested loop I receive the below error:
MySql.Data.MySqlClient.MySqlException was unhandled
Message=There is already an open DataReader associated with this Connection which must be closed first.
Source=MySql.Data
ErrorCode=-2147467259
Number=0
My class currently looks like this
public class MySQLManager
{
private MySqlConnection _MySQLRemoteConnection { get; set; }
public void setup(string remoteUser, string remotePass, string remoteServerAddress, string remoteDb, string localUser, string localPass, string localServerAddress, string localDb)
{
_remote_server_address = remoteServerAddress;
_remote_database = remoteDb;
_remote_username = remoteUser;
_remote_password = remotePass;
}
public void connect()
{
try
{
_MySQLRemoteConnection = new MySqlConnection() { ConnectionString = string.Format("server={0};database={1};uid={2};password={3};", _remote_server_address, _remote_database, _remote_username, _remote_password) };
_MySQLRemoteConnection.Open();
_RemoteConnection = true;
}
catch (MySqlException ex)
{
_RemoteConnection = false;
}
}
public MySqlCommand run(string query, List<MySqlParameter> dbparams = null)
{
connect();
MySqlCommand sql = getConnection().CreateCommand();
sql.CommandText = query;
if (dbparams != null)
{
if (dbparams.Count > 0)
{
sql.Parameters.AddRange(dbparams.ToArray());
}
}
//disconnect();
return sql;
}
public MySqlDataReader fetch(MySqlCommand cmd)
{
//connect();
var t = cmd.ExecuteReader();
//disconnect();
return t;
}
And the code that I'm running to create the error, now I understand I can do the below example in a single query, this is an EXAMPLE query to re-create the error, writing it into a single query will not work with live examples.
query = "SELECT field1 FROM tmp WHERE field1 < 3";
using (var sql = db.run(query))
{
txtResponse.Text += "Query ran" + nl;
using (var row = db.fetch(sql))
{
txtResponse.Text += "Query fetched" + nl;
db.connect();
while (row.Read())
{
txtResponse.Text += "Row : " + row[0].ToString() + nl;
query = "SELECT val1 FROM tmp2 WHERE field1 = '" + row[0].ToString() + "'";
//db.disconnect();
using (var sql2 = db.run(query))
{
txtResponse.Text += "Query ran" + nl;
db.disconnect();
using (var row2 = db.fetch(sql))
{
txtResponse.Text += "Query fetched" + nl;
db.connect();
while (row.Read())
{
txtResponse.Text += " Val : " + row2[0].ToString() + nl;
}
}
}
}
}
}
So how would I go about getting the second loop to work?
For SQL Server, you could use MultipleActiveResultSets=true on connection string, but this most likely won't work for MySQL.
The other option is to use 2 connections, one for each data reader.