I'm at a complete loss here. I have two tests: Update_Task and Get_All_Tasks. When run individually, they pass. If I run them together in Visual Studio 2015 (not in parallel -- just selecting them both and clicking run) Update_Task passes and Get_All_Tasks does not. For the latter, I get an error message saying that the database is locked. This implies that Update_Task locks the database and fails to release the lock, but I can't figure out where that would do it.
Any thoughts? Code follows.
EDIT: I found the answer.
It turns out that when you do a var reader = cmd.ExecuteReader(), that reader needs to be put in a using block itself. Otherwise, the reader may not close and that caused the lock. Doing this fixed the problem:
using (var reader = cmd.ExecuteReader())
{
// The query will return only one item, so read it into a new task.
reader.Read();
return this.GetTaskFromReader(reader);
}
Test Methods
[TestMethod]
public void Update_Task()
{
var dbMan = new DatabaseManager();
// Add a new task
var t = new Task()
{
IsEnabled = true,
Description = "Test",
DueDate = DateTime.Now,
Priority = Priority.High
};
dbMan.InsertTask(t);
// Alter the task and update it
t.IsEnabled = false;
t.Description = "Updated test";
t.DueDate = DateTime.Now.AddDays(1);
t.Priority = Priority.Medium;
var updatedTask = this.UpdateTaskAndRetrieve(dbMan, t);
Assert.AreEqual(t, updatedTask, "Task did not update correctly.");
}
[TestMethod]
public void Get_All_Tasks()
{
var dbMan = new DatabaseManager();
// Get the number of tasks currently in the database
var originalCount = dbMan.GetAllTasks().Count;
// Add 5 tasks with different values
for (int i = 0; i < 5; i++)
{
var t = new Task()
{
IsEnabled = i % 2 == 0,
Description = $"Task {i + 1}",
DueDate = DateTime.Now.AddDays(i),
Priority = (Priority)(i % 4)
};
dbMan.InsertTask(t); // <-- Fails when i == 0
}
// Get all the tasks in the database.
var result = dbMan.GetAllTasks();
Assert.AreEqual(originalCount + 5, result.Count, "Wrong number of tasks came back from the database.");
}
private Task UpdateTaskAndRetrieve(DatabaseManager dbMan, Task t)
{
dbMan.UpdateTask(t);
return dbMan.GetTask(t.Id);
}
DatabaseManager Methods
public void UpdateTask(Task task)
{
if (task.Id == 0)
{
throw new InvalidOperationException("Cannot update a task that has not yet been inserted.");
}
using (var connection = new SQLiteConnection(_connectionString))
{
connection.Open();
using (var cmd = new SQLiteCommand(connection))
{
var sql = #"UPDATE [Task]
SET
[enabled]=#enabled,
[description]=#description,
[duedate]=#duedate,
[priority]=#priority
WHERE rowid=#rowId";
// Build up the command and its parameters
cmd.CommandType = CommandType.Text;
cmd.CommandText = sql;
cmd.Parameters.Add(new SQLiteParameter("#enabled", task.IsEnabled ? 1 : 0));
cmd.Parameters.Add(new SQLiteParameter("#description", task.Description));
cmd.Parameters.Add(new SQLiteParameter("#duedate", task.DueDate.Ticks));
cmd.Parameters.Add(new SQLiteParameter("#priority", (int)task.Priority));
cmd.Parameters.Add(new SQLiteParameter("#rowid", task.Id));
var result = cmd.ExecuteNonQuery();
Debug.WriteLine($"Command executed. {result} rows affected.");
}
}
}
public IList<Task> GetAllTasks()
{
using (var connection = new SQLiteConnection(_connectionString))
{
connection.Open();
using (var cmd = new SQLiteCommand(connection))
{
// Create the command
var sql = "SELECT rowid, * from [Task] ORDER BY [duedate] DESC, [priority] DESC";
cmd.CommandText = sql;
// Execute the command and build the resultant list
var reader = cmd.ExecuteReader();
var retList = new List<Task>();
while (reader.Read())
{
retList.Add(this.GetTaskFromReader(reader));
}
return retList;
}
}
}
public Task GetTask(int id)
{
using (var connection = new SQLiteConnection(_connectionString))
{
connection.Open();
using (var cmd = new SQLiteCommand(connection))
{
var sql = "SELECT rowid, * from [Task] WHERE rowid=#id";
cmd.CommandText = sql;
cmd.Parameters.Add(new SQLiteParameter("#id", id));
var reader = cmd.ExecuteReader();
// The query will return only one item, so read it into a new task.
reader.Read();
var task = this.GetTaskFromReader(reader);
return task;
}
}
}
private Task GetTaskFromReader(SQLiteDataReader reader)
{
var retId = Convert.ToInt32(reader["rowid"]);
var desc = reader["description"].ToString();
var date = new DateTime((long)reader["duedate"]);
var isEnabled = Convert.ToBoolean(reader["enabled"]);
var priority = (Priority)Enum.Parse(typeof(Priority), reader["priority"].ToString());
return new Task(retId, isEnabled, desc, date, priority);
}
Related
trying to update a column of null values with instagramIds, this is my current approach but the console app just keeps running and doesn't update any values in the database.
public static async Task<InstagramUser> ScrapeInstagram(string url)
{
using (var client = new HttpClient())
{
var response = await client.GetAsync(url);
if (response.IsSuccessStatusCode)
{
// create html document
var htmlBody = await response.Content.ReadAsStringAsync();
var htmlDocument = new HtmlDocument();
htmlDocument.LoadHtml(htmlBody);
// select script tags
var scripts = htmlDocument.DocumentNode.SelectNodes("/html/body/script");
// preprocess result
var uselessString = "window._sharedData = ";
var scriptInnerText = scripts[0].InnerText
.Substring(uselessString.Length)
.Replace(";", "");
// serialize objects and fetch the user data
dynamic jsonStuff = JObject.Parse(scriptInnerText);
dynamic userProfile = jsonStuff["entry_data"]["ProfilePage"][0]["graphql"]["user"];
List<String> columnData = new List<String>();
//Update database query
string connectionString = #"Server=myProject-dev-db.cothtpanmcn7.ap-southeast-2.rds.amazonaws.com;Database=Projectdb;User Id=testadmin;Password=U8gs7vb7C7yvakXf;MultipleActiveResultSets=true;Trusted_Connection=False;";
using (SqlConnection con = new SqlConnection(connectionString))
{
//get null values from database
string query = "Select * from ApplicationUser where InstagramId is null";
using (SqlCommand command = new SqlCommand(query, con))
{
command.Connection.Open();
using (SqlDataReader reader = command.ExecuteReader())
{
while (reader.Read())
{
columnData.Add(reader.GetString(0));
}
}
}
for (int index = 0; index < columnData.Count(); index++)
{
//get username and scrape info
var instagramInfo = new InstagramUser
{
Id = userProfile.id,
};
columnData.Add(instagramInfo.ToString());
}
SqlCommand cmd = new SqlCommand("Update ApplicationUser Set InstagramId = '" + columnData + "'" + "where InstagramUsername = '" + userprofile.username + "'", con);
cmd.Connection.Open();
cmd.ExecuteNonQuery();
}
// create an InstagramUser
var instagramUser = new InstagramUser
{
FullName = userProfile.full_name,
FollowerCount = userProfile.edge_followed_by.count,
FollowingCount = userProfile.edge_follow.count,
Id = userProfile.id,
url = url
};
return instagramUser;
}
else
{
throw new Exception($"Something wrong happened {response.StatusCode} - {response.ReasonPhrase} - {response.RequestMessage}");
}
}
}
The current output:
{"FullName":null,"FollowerCount":0,"FollowingCount":0,"Id":"6978701146","url":null}
{"FullName":null,"FollowerCount":0,"FollowingCount":0,"Id":"6978701146","url":null}
{"FullName":null,"FollowerCount":0,"FollowingCount":0,"Id":"6978701146","url":null}
{"FullName":null,"FollowerCount":0,"FollowingCount":0,"Id":"6978701146","url":null}
{"FullName":null,"FollowerCount":0,"FollowingCount":0,"Id":"6978701146","url":null}
My current approach is to create a list, add all instagramIDs which are null to that list. From there I add all instagramIds to that list after scraping Instagram for their usernames e.g. https://www.instagram.com/therock/?__a=1
then I update the column InstagramUsername with their instagram Ids
ExecuteReader has rows but initially when i hover and look at result view it has question marks for all the rows. Then when i go back in a second time to hover it says enumeration yielded no results. the HasRows is true however. I'm also getting my output parameters fine, but not the result set from the select statement. I'm executing a sql server stored proc. the SP executes fine in management studio. And again i am getting the output parms, just not the result set.
internal static CrossWalk Create(string senderId, string ediType)
{
var connection = ConfigurationManager.ConnectionStrings["caeCustom"];
DbCommand cmd = new SqlCommand();
cmd.CommandText = "emb_edi_xwalk_select";
cmd.CommandType = CommandType.StoredProcedure;
cmd.AddParameter("#pSENDER_ID", senderId);
cmd.AddParameter("#pMAPPING_TYPE", ediType);
var delegateId = new SqlParameter
{
ParameterName = "#pDELEGATE_ID",
SqlDbType = SqlDbType.Int,
Direction = ParameterDirection.Output
};
cmd.Parameters.Add(delegateId);
var crossWalk = new CrossWalk();
var dbFactory = DbProviderFactories.GetFactory(connection.ProviderName);
using(var sqlConnection =
dbFactory.CreateConnection(connection.ConnectionString))
{
cmd.Connection = sqlConnection;
using (var sqlReader = cmd.ExecuteReader())
{
if (sqlReader.HasRows)
{
while (sqlReader.Read())
{
//Mapping of AAA values will be AAA03~internal_value -->
loop~external_value
//i.e. AAA03~1008 --> 2010EA~51
string key;
string value;
if (sqlReader["element"].ToString() == "AAA03")
{
key = string.Join("~",
sqlReader["element"].ToString(), sqlReader["internal_value"].ToString());
value = string.Join("~", sqlReader["loop"].ToString(),
sqlReader["external_value"].ToString());
}
else
//Normal xwalk mapping will be loop+element~external_value
--> internal_value
//i.e. 2010ANM101~X3 --> 3
{
key = string.Join("~", sqlReader["loop"] +
sqlReader["element"].ToString(), sqlReader["external_value"].ToString());
value = sqlReader["internal_value"].ToString();
}
crossWalk._lookups.Add(key, value);
}
}
sqlReader.Close();
}
crossWalk.DelegateId =
Convert.ToInt32(cmd.Parameters["#pDELEGATE_ID"].Value);
}
return crossWalk;
}
I am trying to doing this:
Read a row from an SQLite db (in GetRuleByID() method)
Update the same row that I just read during (1) (See UpdateStatusForRuleID() method)
However my problem is that SQLite locks the database after the SELECT in GetRuleByID() so that update in UpdateStatusForRuleID() is only successful when called the first time.
I have tried enabling Write-Ahead-Logging in SQLite as well as PRAGMA read_uncommitted=1 in order to avoid SQLite locking the database for the SELECT, but this does not appear to work.
This should be simple but I have so far spent a complete night trying to solve this... Please help !
private static MicroRuleEngine.Rule GetRuleByID(int ruleID, SQLiteConnection connection, out Dictionary<string, string> dict)
{
dict = new Dictionary<string, string>();
string sql = String.Format("select * from rules WHERE ID = {0} ", ruleID.ToString());
SQLiteCommand command = new SQLiteCommand(sql, connection);
SQLiteDataReader reader = command.ExecuteReader();
if (reader.HasRows)
{
reader.Read();
// Convert row into a dictionary
for (int lp = 0; lp < reader.FieldCount; lp++)
{
dict.Add(reader.GetName(lp), reader.GetValue(lp) as string);
}
string json = dict["fulljson"];
MicroRuleEngine.Rule r = Newtonsoft.Json.JsonConvert.DeserializeObject<MicroRuleEngine.Rule>(json);
//command.Dispose();
return r;
}
}
internal static void UpdateStatusForRuleID(SQLConnectionManager DBMANAGER, int ruleID, bool status)
{
Dictionary<string, string> dict = null;
string dbVal = (status) ? "1" : "0";
MicroRuleEngine.Rule r = null;
string newJSON = null;
using (SQLiteConnection connection = DBMANAGER.CreateConnection())
{
r = GetRuleByID(ruleID, connection, out dict);
r.Active = (status);
newJSON = Newtonsoft.Json.JsonConvert.SerializeObject(r);
Thread.Sleep(1000);
string sql = "UPDATE rules SET active = #a, fulljson=#j WHERE ID = #i";
using (var command = new SQLiteCommand(sql, connection))
{
command.Parameters.Add(new SQLiteParameter("#a", dbVal));
command.Parameters.Add(new SQLiteParameter("#i", ruleID));
command.Parameters.Add(new SQLiteParameter("#j", newJSON));
command.ExecuteNonQuery(); // Database is locked here ???
}
connection.Close();
}
}
"Database is locked" means that some other connection (in the same or some other process) still has an active transaction.
You don't need multiple connections (unless you are using multiple threads); just use a single connection object for all database accesses.
Ensure that all command, reader, and transaction objects (and connections, if you decide to use temporary ones) are properly cleaned up, by using using:
using (var command = new SQLiteCommand(sql, connection))
using (var reader = command.ExecuteReader())
{
if (reader.HasRows)
...
}
Apparently, the code below works. I basically dropped the GetRuleByID() method (but then I had to re-write 4 other methods)
Thanks to all who provided input.
internal static void UpdateStatusForRuleID(SQLConnectionManager DBMANAGER, int ruleID, bool status)
{
string dbVal = (status) ? "1" : "0";
MicroRuleEngine.Rule r = null;
string newJSON = null;
using (SQLiteConnection conn = DBMANAGER.CreateConnection())
{
string sql = String.Format("select * from rules WHERE ID = {0} ", ruleID.ToString());
using (var command = new SQLiteCommand(sql, conn))
using (var reader = command.ExecuteReader())
{
if (reader.HasRows)
{
reader.Read();
string json = reader["fulljson"].ToString();
r = Newtonsoft.Json.JsonConvert.DeserializeObject<MicroRuleEngine.Rule>(json);
r.Active = (status);
newJSON = Newtonsoft.Json.JsonConvert.SerializeObject(r);
string sql2 = "UPDATE rules SET active = #a, fulljson=#j WHERE ID = #i";
using (var command2 = new SQLiteCommand(sql2, conn))
{
command2.Parameters.Add(new SQLiteParameter("#a", dbVal));
command2.Parameters.Add(new SQLiteParameter("#i", ruleID));
command2.Parameters.Add(new SQLiteParameter("#j", newJSON));
command2.ExecuteNonQuery();
}
}
}
}
}
I have a StoredProcedure that I created like this;
CREATE DEFINER=`mysqladmin`#`%` PROCEDURE `Alerts_GetAlerts`(IN managerID INT)
BEGIN
SELECT ID, Type, EmpID, ManagerID, HolID
FROM Alerts
WHERE ManagerID = managerID;
END$$
I then try to call this from my C# code like so;
using (var con = new MySqlConnection(MySQLConStr))
{
con.Open();
using (MySqlCommand cmd = new MySqlCommand("Alerts_GetAlerts", con))
{
cmd.Parameters.AddWithValue("#managerID", managerID);
using (var dataReader = cmd.ExecuteReader())
{
while (dataReader.Read())
{
var alert = new AlertsModel
{
ID = Convert.ToInt32(dataReader[0]),
Type = Convert.ToInt32(dataReader[1]),
ManagerID = Convert.ToInt32(dataReader[2]),
EmployeeID = Convert.ToInt32(dataReader[3]),
HolidayID = Convert.ToInt32(dataReader[4]),
};
AllAlerts.Add(alert);
}
}
}
return AllAlerts;
}
However I constantly get, Incorrect number of arguments for PROCEDURE sdcdatabase.Alerts_GetAlerts; expected 1, got 0 even though to me it appears I am passing the managerID argument through;
cmd.Parameters.AddWithValue("#managerID", managerID);
Where am I going wrong?
try this code
using (var con = new MySqlConnection(MySQLConStr))
{
con.Open();
using (MySqlCommand cmd = new MySqlCommand("Alerts_GetAlerts(#managerID)", con))
{
cmd.Parameters.AddWithValue("#managerID", managerID);
using (var dataReader = cmd.ExecuteReader())
{
while (dataReader.Read())
{
var alert = new AlertsModel
{
ID = Convert.ToInt32(dataReader[0]),
Type = Convert.ToInt32(dataReader[1]),
ManagerID = Convert.ToInt32(dataReader[2]),
EmployeeID = Convert.ToInt32(dataReader[3]),
HolidayID = Convert.ToInt32(dataReader[4]),
};
AllAlerts.Add(alert);
}
}
}
return AllAlerts;
}
I'm working on an autocomplete script.
When user stop typing, his input is send to the server which will look for some matches in DB.
Eg: If looking for people "Obam" should return "Barack Obama".
I want this search to be limited to ~500ms. If it takes longer, I want to abort the search and only return results found in time.
I start by looking for a perfect match (this one won't be interrupted), then I am looking for a partial match (this one can be interrupted) :
private static MySqlConnection conn = new MySqlConnection(ConfigurationManager.AppSettings["CxMySql"].ToString());
protected void Page_Load(object sender, EventArgs e)
{
conn.Open();
SearchTerm(table,term,domaine);
conn.Close(); // TimeoutException HERE
conn.Dispose();
}
private void SearchTerm(string table,string term,string domaine)
{
Dictionary<int, Scoring> results = new Dictionary<int, Scoring>();
var requete = "SELECT m.id, m.nom as label FROM marque m WHERE m.nom = '" + term + "'";
var Cmd = new MySqlCommand(requete, conn);
using (var Rd = Cmd.ExecuteReader())
{
while (Rd.Read())
{
results.Add(int.Parse(Rd["id"].ToString()), new Scoring()
{
score = 1000,
value = Rd["label"].ToString()
});
}
}
// Here it should be 500, but I put 1 to force troubles to appear.
RunWithTimeout(() => FindOtherBrands(term, ref results), TimeSpan.FromMilliseconds(1));
var resultsList = results.ToList();
resultsList.Sort(
delegate(KeyValuePair<int, Scoring> firstPair,
KeyValuePair<int, Scoring> nextPair)
{
return nextPair.Value.score - firstPair.Value.score;
}
);
}
private void FindOtherBrands(string term, ref Dictionary<int, Scoring> results)
{
MySqlCommand Cmd;
string requete;
requete = "SELECT m.id, m.nom as label FROM marque m WHERE m.nom LIKE '" + term + "%'";
Cmd = new MySqlCommand(requete, conn);
var Rd = Cmd.ExecuteReader(); // NullReferenceException HERE
while (Rd != null && Rd.Read())
{
int id = int.Parse(Rd["id"].ToString());
if (!results.ContainsKey(id))
{
results.Add(id, new Scoring()
{
score = 100,
value = Rd["label"].ToString()
});
}
}
Rd.Close();
requete = "SELECT m.id, m.nom as label FROM marque m WHERE m.nom LIKE '%" + term + "%'";
Cmd = new MySqlCommand(requete, conn);
Rd = Cmd.ExecuteReader();
while (Rd != null && Rd.Read())
{
int id = int.Parse(Rd["id"].ToString());
if (!results.ContainsKey(id))
{
results.Add(id, new Scoring()
{
score = 10,
value = Rd["label"].ToString()
});
}
}
Rd.Close();
}
I found the RunWithTimeout method here : stop executing code in thread after 30s
bool RunWithTimeout(ThreadStart threadStart, TimeSpan timeout)
{
Thread workerThread = new Thread(threadStart);
workerThread.Start();
bool finished = true;
if (!workerThread.Join(timeout))
{
workerThread.Abort();
finished = false;
}
return finished;
}
Scoring is a struct for an easy sorting of results
private struct Scoring
{
public string value;
public int score;
}
The aim is to have results (not necessarily all) fast.
PROBLEMS
I randomly get Timeoutexception after ~30s on the conn.Close(); line.
I randomly get NullReferenceException on the first Cmd.ExecuteReader(); call in FindOtherBrands.
Can anyone explain me why ?
Am I doing something wrong or is there a workaround ?
I guess the TimeoutException is because I am trying to close connection during command execution, can I drop/cancel that query ?
I would take a different approach. Since you're querying a database, which is naturally asynchronous, you can use async-await for querying the data. With that, you can pass a CancellationToken which is set to a timeout, which you'll monitor with every read:
For example:
private async Task FindOtherBrands(string term,
Dictionary<int, Scoring> results,
CancellationToken cancellationToken)
{
MySqlCommand cmd;
string requete;
requete = "SELECT m.id, m.nom as label
FROM marque m
WHERE m.nom LIKE '" + term + "%'";
cmd = new MySqlCommand(requete, conn);
var Rd = await cmd.ExecuteReaderAsync();
while (Rd != null && await Rd.ReadAsync())
{
cancellationToken.ThrowIfCancellationRequested();
int id = int.Parse(Rd["id"].ToString());
if (!results.ContainsKey(id))
{
results.Add(id, new Scoring()
{
score = 100,
value = Rd["label"].ToString()
});
}
}
Rd.Close();
requete = "SELECT m.id, m.nom as label
FROM marque m
WHERE m.nom LIKE '%" + term + "%'";
cmd = new MySqlCommand(requete, conn);
Rd = await Cmd.ExecuteReaderAsync();
while (Rd != null && await Rd.ReadAsync())
{
cancellationToken.ThrowIfCancellationRequest();
int id = int.Parse(Rd["id"].ToString());
if (!results.ContainsKey(id))
{
results.Add(id, new Scoring()
{
score = 10,
value = Rd["label"].ToString()
});
}
}
Rd.Close();
}
And when you invoke it, all you need is to wrap it in a try-catch and pass a CancellationToken:
private async Task<bool> RunWithTimeoutAsync(TimeSpan timeout)
{
bool finished;
try
{
var cancellationTokenSource = new CancellationTokenSource(timeout);
await FindOtherBrandsAsnyc(term,
results,
cancellationTokenSource.CancellationToken);
finished = true;
}
catch (OperationCanceledException e)
{
// Handle
}
return finished;
}
Side note - Your query is prone to SQL Injection. You shouldn't used string concatenation. Use query parameters instead.