I'm a VFP developer thats trying to pick up C#. Figuring that the fastest way to learn is by doing, I gave myself a small project for practice.
Objective
Move some transactions from the "live" database into an archive database.
The general idea was:
to get a list of "suitable" Jobs (parent) then to read() them one by
one.
use a foreach loop to process a list of child tables, inserting the
new record in the History database followed by deleting the old one
from the "live" table.
move the actual Job (parent) to archive, then delete it from "live"
Things went OK at the start, then I suddenly hit a brick wall....
Problem
I have a variable called RefCombo that I used to store the parent's primary key, this is stored right after the while (READ()) statement and just before the foreach loop.
In a test database of four records, the RefCombo field should have been:
while read() result: Job1
foreach childtable in my predefined list
process ChildTable1
process ChildTable2
process ChildTable3
process ChildTable4
process ChildTable5
then repeating with Job2, Job3 and Job4.
What's driving me nuts is I'm getting this instead:
while read() result: Job1
foreach childtable in my predefined list
process ChildTable1
SKIP other ChildTables, RefCombo (primary key) becomes Job2, loops back to the while read()
The thing is, once it changes to Job2, it proceeds to work exactly like I expected, looping through every single child table for each remaining parent record.
I'm also getting a "The current TransactionScope is already complete." error every other time I'm running the web page. This happens at the connection.open() part at the start of the program but it's hard to pin down since it doesn't happen all the time.
I've posted the code at the bottom, its sorta longish for a casual glance but any help would be much appreciated. I've been working on this for 2 days and really can't afford any more hair loss, lol.
Any comments on my code would also be more that welcome, I'm REALLY new to C# and every bit helps :P
Thanks in advance.
#region Declaration
DateTime FromDate = Convert.ToDateTime("01/01/2011");
DateTime ToDate = Convert.ToDateTime("01/03/2011");
string conStrSource = #"Data Source=HOME\SQLEXPRESS;Initial Catalog=MCC_Live;Integrated Security=True";
// Declare a list of child tables to check and move together with the Job
List<String> TablesToMove = new List<String>();
{
TablesToMove.Add("JobItems");
TablesToMove.Add("JobTerms");
TablesToMove.Add("JobMessages");
TablesToMove.Add("JobCalcs");
TablesToMove.Add("JobContainers");
TablesToMove.Add("JobMetrics");
}
#endregion
#region Set SQL String
string QueryString =
"SELECT " +
"JobMaster.BranchID, JobMaster.JobNo, " +
"JobMaster.ImportDate, PermitStatus.CurrentStatus " +
"FROM JobMaster. " +
"INNER JOIN PermitStatus ON " +
"JobMaster.BranchID = PermitStatus.BranchID AND " +
"JobMaster.JobNo = PermitStatus.JobNo " +
"WHERE " +
"(JobMaster.ImportDate BETWEEN #FromDate AND #ToDate) AND " +
"PermitStatus.currentStatus NOT IN ('HT0', 'HT1', 'HT2')";
#endregion
// Display on web page for reference
ASPxFromDate.Value = FromDate;
ASPxToDate.Value = ToDate;
ASPxMemo1.Value = QueryString;
#region Open Connection, Get List of filtered Master Jobs
using (SqlConnection connection = new SqlConnection(conStrSource))
{
int JobCount = 0;
ASPxListBox1.Items.Clear();
ASPxListBox2.Items.Clear();
ASPxListBox3.Items.Clear();
ASPxListBox1.Columns.Clear();
SqlCommand command = new SqlCommand(QueryString, connection);
{
command.Parameters.Add(new SqlParameter("#FromDate", FromDate));
command.Parameters.Add(new SqlParameter("#ToDate", ToDate));
}
connection.Open();
SqlDataReader FilteredJobList = command.ExecuteReader();
#endregion
try // Process MasterJob File
{
// Process individual jobs one by one so I won't tie up memory and have better logging
while (FilteredJobList.Read())
{
#region Declare variables
string RefCombo = (string)FilteredJobList[0] + (string)FilteredJobList[1]; //Get primary key
JobCount = JobCount + 1;
ASPxTextBox2.Value = JobCount;
ASPxListBox2.Items.Add(RefCombo);
#endregion
// Start transaction scope
TransactionScope TranScope = new TransactionScope();
{
try
{
// Loop through child tables
foreach (string CurrentTable in TablesToMove)
{
#region Transfer child tables
// update list so I know which part its working on
ASPxListBox1.Items.Add(CurrentTable);
RefCombo = (string)FilteredJobList[0] + (string)FilteredJobList[1];
string RefTableNow = (string)CurrentTable;
bool CancelTrans = false;
MoveChild(ref RefCombo, ref RefTableNow, ref conStrSource, ref CancelTrans);
if (CancelTrans == false)
{ //LogFailure();
break;
}
DelChild(ref RefCombo, ref RefTableNow, ref conStrSource, ref CancelTrans);
if (CancelTrans == false)
{ //LogFailure();
break;
}
#endregion
// Remove remaing entries
//MoveLatestStatus();
//DeleteLatestStatus();
//MoveMasterJob();
//DeleteMasterJob();
//LogSuccess();
TranScope.Complete();
}
catch
{
//LogFailure();
}
}
}
finally
{
FilteredJobList.Close();
}
}
}
//------------------------------------------------------------------------------------------------
private void MoveChild(ref string RefCombo, ref string CurrentTable, ref string conStrSource, ref bool CancelTrans)
{
#region Define Insert String
string InsertSqlString =
"INSERT INTO [MCC_History].[dbo].[" + #CurrentTable + "]" +
" SELECT * FROM [MCC_Live].[dbo].[" + #CurrentTable + "] s" +
" WHERE NOT EXISTS" +
" (SELECT 1 FROM [MCC_History].[dbo].[" + #CurrentTable + "] t2" +
" WHERE t2.BranchID + t2.JobNo = s.BranchID + s.JobNo)" +
" AND s.BranchID + s.JobNo = #RefCombo";
#endregion
#region Open connection and execute query
using (SqlConnection MoveConnect = new SqlConnection(conStrSource))
{
try
{
SqlCommand InsertCommand = new SqlCommand(InsertSqlString, MoveConnect);
{
InsertCommand.Parameters.Add(new SqlParameter("#RefCombo", RefCombo));
}
MoveConnect.Open();
InsertCommand.ExecuteNonQuery();
}
catch
{
CancelTrans = true;
}
}
#endregion
}
//------------------------------------------------------------------------------------------------
private void DeleteChild(ref string RefCombo, ref string CurrentTable, ref string conStrSource, ref bool CancelTrans)
{
#region Define Delete query
string DeleteSqlString =
" DELETE FROM [MCC_Live].[dbo].[" + #CurrentTable + "]" +
" WHERE [MCC_Live].[dbo].[" + #CurrentTable +
"].BranchID + [MCC_DB].[dbo].[" + #CurrentTable + "].JobNo = #RefCombo";
#endregion
#region Execute Delete query
using (SqlConnection MoveConnect = new SqlConnection(conStrSource))
{
try
{
SqlCommand InsertCommand = new SqlCommand(DeleteSqlString, MoveConnect);
{
InsertCommand.Parameters.Add(new SqlParameter("#RefCombo", RefCombo));
}
MoveConnect.Open();
InsertCommand.ExecuteNonQuery();
}
catch
{
CancelTrans = true;
}
}
#endregion
}
The problem is where you have your TranScope.Complete(); statement. It is inside of your foreach loop and it belongs after your Foreach block exits. You complete your transaction at the end of your first iteration of your list. This prevents you from making any more commands through this transaction.
Related
I am trying to make an update on my MySql database, but I don't understand why it doesn't work when executing from MySqlCommand, I have the famous message "check the manual that corresponds to your MySQL server version" (I already have hundreds of queries working correctly, so I guess something is with syntax?).
edit 2 : Here is the part "near " :
'В-513',PRIORITY=1050,QUANTITY_INIT=28,QUANTITY_REMAINING=0,FICHIER='C:\\Actcut3' at line 1'
Here is the update query :
UPDATE launching_order_details SET
ID_LO=1935,
ID_CONTRACT=4228,
ID_PHASE=11765,
ID_ASS=235314,
LIST_REP_ORI='1005817//В-513//235314//В1007//11765//1//30',IS_SUBDETAIL=0,
REF_DETAIL='3201\\1\\В1007\\В-513\\',
NAME='В-513',
PRIORITY=1050,QUANTITY_INIT=28,QUANTITY_REMAINING=0,
FICHIER='C:\\Actcut3.10\\Data\\Parts\\3201\\1\\В1007\\В-513.ini' WHERE ID=27701
Of course I send it in a same line, I just splitted it here for better readability.
If I make a copy/paste of query, then execute it from phpMyadmin, all is working fine.
Edit : C# code :
DBConnect class :
public void Update(string query)
{
if (this.OpenConnection() == true)
{
if (isMySQL)
{
MySqlCommand cmd = new MySqlCommand(query.Replace("[vsteel].", ""), connection);
cmd.ExecuteNonQuery();
this.CloseConnection();
}
else
{
SqlCommand command = new SqlCommand(query, MSconnection);
command.Parameters.Add(new SqlParameter("0", 1));
//command.Connection = this.MSconnection;
command.ExecuteNonQuery();
this.CloseConnection();
}
}
}
public bool OpenConnection()
{
if (isMySQL)
{
try
{
connection.Open();
return true;
}
catch (MySqlException ex)
{
//When handling errors, you can your application's response based
//on the error number.
//The two most common error numbers when connecting are as follows:
//0: Cannot connect to server.
//1045: Invalid user name and/or password.
switch (ex.Number)
{
case 0:
MessageBox.Show("Cannot connect to server. Contact administrator");
break;
case 1045:
MessageBox.Show("Invalid username/password, please try again");
break;
}
System.Windows.Forms.Application.Exit();
Global.is_restarted = true;
return false;
}
}
else
{
try
{
MSconnection.Open();
return true;
}
catch (MySqlException ex)
{
//When handling errors, you can your application's response based
//on the error number.
//The two most common error numbers when connecting are as follows:
//0: Cannot connect to server.
//1045: Invalid user name and/or password.
switch (ex.Number)
{
case 0:
MessageBox.Show("Cannot connect to server. Contact administrator");
break;
case 1045:
MessageBox.Show("Invalid username/password, please try again");
break;
}
System.Windows.Forms.Application.Exit();
Global.is_restarted = true;
return false;
}
}
}
RepereLO class :
private void update()
{
this.listRepereOri = this.listRepereOri.OrderBy(x => x.Priority).ThenBy(x => x.ID).ToList();
DBConnect DataBase = new DBConnect();
string query = "UPDATE [vsteel].launching_order_details SET " +
"ID_LO=" + this.launchingOrder.ID + "," +
"ID_CONTRACT=" + this.contract.ID + "," +
"ID_PHASE=" + this.phase.ID + "," +
"ID_ASS=" + this.assembly.ID + "," +
"LIST_REP_ORI=\'" + convertListRepereOriToString() + "\'," +
"IS_SUBDETAIL=" + Convert.ToInt32(this.isSubRepere) + "," +
"REF_DETAIL=\'" + this.refDetail + "\'," +
"NAME=\'" + this.name + "\'," +
"PRIORITY=" + this.priority + "," +
"QUANTITY_INIT=" + this.quantity + "," +
"QUANTITY_REMAINING=" + this.remainingQuantity + "," +
"FICHIER=\'" + Global.ReplaceSpecialCharacters(this.fileName) + "\' " +
"WHERE ID=" + this.id;
DataBase.Update(query);
}
EDIT 2 : Parametirezed query
my DBConnect class
public void UpdateNew(string query, MySqlParameter[] myParamArray)
{
if (this.OpenConnection() == true)
{
using (MySqlCommand cmd = new MySqlCommand(query.Replace("[vsteel].", ""), connection))
{
for (int i = 0; i < myParamArray.Count(); i++)
{
cmd.Parameters.Add(myParamArray[i]);
}
cmd.Prepare();
cmd.ExecuteNonQuery();
}
}
}
In object :
private void update()
{
this.listRepereOri = this.listRepereOri.OrderBy(x => x.Priority).ThenBy(x => x.ID).ToList();
string query = "UPDATE [vsteel].launching_order_details SET " +
"ID_LO=#idLo," +
"ID_CONTRACT=#idContract," +
"ID_PHASE=#idPhase," +
"ID_ASS=#idAss," +
"LIST_REP_ORI=#listRepOri," +
"IS_SUBDETAIL=#isSubdetail," +
"REF_DETAIL=#refDetail," +
"NAME=#name," +
"PRIORITY=#priority," +
"QUANTITY_INIT=#qtyInit," +
"QUANTITY_REMAINING=#qtyRemaining," +
"FICHIER=#fichier" +
" WHERE ID=#id";
MySqlParameter[] listParams = new MySqlParameter[]
{
new MySqlParameter("id", this.id),
new MySqlParameter("idLo", this.launchingOrder.ID),
new MySqlParameter("idContract", this.Contract.ID),
new MySqlParameter("idPhase", this.Phase.ID),
new MySqlParameter("idAss", this.Assembly.ID),
new MySqlParameter("listRepOri", this.convertListRepereOriToString()),
new MySqlParameter("isSubdetail", this.isSubRepere),
new MySqlParameter("refDetail", this.refDetail),
new MySqlParameter("name", this.name),
new MySqlParameter("priority", this.priority),
new MySqlParameter("qtyInit", this.quantity),
new MySqlParameter("qtyRemaining", this.remainingQuantity),
new MySqlParameter("fichier", this.fileName),
};
DBConnect DataBase = new DBConnect();
DataBase.UpdateNew(query, listParams);
}
The actual problem is using string concatenation to construct a query from external input. This leaves the code wide open to SQL injection, conversion errors (what date format? decimal separator?) and ... syntax errors like this. What if Name is O'Reily for example? Or a user entered ' DROP TABLE Students; # ? No amount of escaping or replacing is going to fix the real bug - using string concatenation.
The correct way to do this is to use parameterized queries. This is actually easier than concatenating strings. If you use a library like Dapper, it's as easy as :
string sql=#"UPDATE [vsteel].launching_order_details
SET
ID_LO=#idlo,
ID_CONTRACT=#contract,
ID_PHASE=#phase,
ID_ASS=#assembly,
LIST_REP_ORI=#ori,
IS_SUBDETAIL=#isSubDetail,
REF_DETAIL=#ref,
NAME=#name,
PRIORITY=#priority,
QUANTITY_INIT=#initial,
QUANTITY_REMAINING=#remaining,
FICHIER=#path,
WHERE ID=#id";
using(var connection=new MySqlConnection(...))
{
connection.Execute(sql, new {
id,
idLo=launchingOrder.ID ,
contract=contract.ID,
....,
path=fileName});
}
Without Dapper, the code is a bit more complex but still easier and safer to write than string concatenation and trying to replace characters.
using(var connection=new MySqlConnection)
using (var cmd=new MySqlCommand(query,connection))
{
cmd.Parameters.AddWithValue("#id",this.id);
...
connection.Open();
cmd.ExecuteNonQuery();
}
BTW the DbConnect class has other issues as well. Long-lived database connections are a bug that harms performance and scalability. The locks taken during a connection remain active until it closes, which results in increased blocking for all clients. This happens even in databases with multi-version concurrency like PostgreSQL.
Connections are meant to be opened as late as possible and closed immediately after use. That's why you see all samples and tutorials create connections in a using block. This ensures the connection is close immediately after use.
ADO.NET uses connection pooling to eliminate the cost of opening a new connection, by reseting existing connections. When DbConnection.Close is called, the connection is reset and placed in a connection pool.
Tutorials
Basics of ADO.NET is a short intro to ADO.NET that explains what the various classes do and how they're used.
MySQL's Tutorial: An Introduction to Connector/NET Programming shows how to use ADO.NET with MySQL.
Microsoft's documentation on ADO.NET is almost an entire book that goes in great depth, so you should probably use it only as a reference
Dapper is a micro-ORM library that makes it very easy to map object properties to parameters and results to objects. It can be used with any ADO.NET provider, including MySQL.
With Dapper, one can write code like this :
public class Dog
{
public int? Age { get; set; }
public Guid Id { get; set; }
public string Name { get; set; }
public float? Weight { get; set; }
public int IgnoredProperty { get { return 1; } }
}
var guid = Guid.NewGuid();
var dog = connection.Query<Dog>("select Age = #Age, Id = #Id", new { Age = (int?)null, Id = guid });
And the library will map the Age and Id properties to #Age and #Id. It will also map the Age and Id columns in the results to Dog.Age and Dog.Id
I am trying to write a method which is supposed to retrieve multiple rows from a table in a database and use those data to instantiate a number of objects. However, as far as i can tell the database only returns the first row. When i do this:
public static List<Event> getMultipleEvents(string[] eventNames)
{
List<Event> rtnList = new List<Event>();
string EventsToRetrieve = "";
foreach (var item in eventNames)
{
if (EventsToRetrieve != "")
{
EventsToRetrieve += " OR ";
}
EventsToRetrieve += "eventName = '";
EventsToRetrieve += item;
EventsToRetrieve += "' ";
}
// This is the string that the method constructs based on the input i am testing with
//"eventName = 'event six' OR eventName = ' event two' OR eventName = ' event one' OR eventName = ' event seven' "
using (SqlConnection sqlConnection = Globals.GetSqlConnection())
{
sqlConnection.Open();
using (SqlCommand sqlCommand = new SqlCommand("SELECT * FROM questions WHERE " + EventsToRetrieve + ";", sqlConnection))
{
using (SqlDataReader sqlDataReader = sqlCommand.ExecuteReader())
{
if (sqlDataReader != null)
{
while (sqlDataReader.Read())
{
Event newEvent = new Event("", DateTime.MinValue, DateTime.MinValue);
string startDateTimeStringFromDB = sqlDataReader["startDateDay"].ToString() + "-" + sqlDataReader["startDateMonth"].ToString() + "-" + sqlDataReader["startDateYear"].ToString();
string endDateTimeStringFromDB = sqlDataReader["endDateDay"].ToString() + "-" + sqlDataReader["endDateMonth"].ToString() + "-" + sqlDataReader["endDateYear"].ToString();
newEvent.EventName = sqlDataReader["eventName"].ToString();
if (DateTime.TryParse(startDateTimeStringFromDB, out DateTime startDateTime))
{
newEvent.StartDate = startDateTime;
}
if (DateTime.TryParse(endDateTimeStringFromDB, out DateTime endDateTime))
{
newEvent.EndDate = endDateTime;
}
rtnList.Add(newEvent);
}
}
}
}
}
return rtnList;
}
Can anyone explain to me what i am doing wrong ? I Also tried wrapping the while loop in a do while loop as suggested here How to read multiple resultset from SqlDataReader? but it didn't change anything.
It doesn't seem to have any error in your code. however I believe that your query has errors.
First of all never ever use string concatenation when you are building your SQL Query. Instead use Parameterized Queries.
with the parameterized queries, it is even easier to debug your SQL statement since it does not include conditional string concatenations.
I have made a small c# procedure that reads a deserialized JSON and iterates it.
While Iterating I have an insert function that inserts each JSON record in a table.
The problem I am facing is while I Itarete the JSON . The records inserted are less than the actual JSON objects.
My code is this
//Code for Iterating
foreach (ContractsAutoView Contract in ConResponce.Contracts)
{ bool Check = AllMethods.ChecKContractExists(ConnectToERP,
Convert.ToString(Contract.ContractNumber));
if (Check != true)
{
int InsResult = AllMethods.InsertContractsToErp(ConnectToERP, Contract);
if (InsResult == 1)
{ RecordResult = RecordResult + "AA: " + Convert.ToString(i) + "Το
συμβόλαιο με Κωδικό " + Contract.ContractNumber + " Ενημέρωσε
την Βάση Δεδομένων " + System.Environment.NewLine;
}
else
{ RecordResult = RecordResult + "AA: " + Convert.ToString(i) + "Το
συμβόλαιο με Κωδικό " + Contract.ContractNumber + " Δεν
Ενημέρωσε την Βάση Δεδομένων " + System.Environment.NewLine;
}
}
i++;
}
//Code for Iterating
// Code for Inserting Record
public static int InsertContractsToErp(SqlConnection Conn, ContractsAutoView contract)
{
int RetRes = 1;
String insertCmd = "Insert into ContractAutoView(ID, BranchID, ContractNumber, EndorsementNumber, RenewalNumber, VoucherPrefix, VoucherNumber, ClientId, Issue, ContractStart, ContractEnd, EffectDate, Status, Notes, NetAmount,FeeAmount,PenaltyAmount ,TotalAmount,PlateNumber, BirthDate, LicenceDate, DurationID, BM, MotorUseID, CoverPacketID, RatingAreaID, PostCode, MotorValue, HP, ConstructionYear, EndorsementType, InsuredName, InsuredAddress, BrandID, Brand, ModelID, Model, TypeID, ColorID, Color, HasABS, HasAirbags, HasAlarm, HasImmobilizer, IsCabrio, IsLiquidGas, IsBrandNew, HasParking, HasHomeOwned, AnnualUse, EmployeeArea, DriversOptions, EngineCC, Weight, WheelBase, ImportToErp) ";
insertCmd = insertCmd + "values(#ID, #BranchID, #ContractNumber, #EndorsementNumber, #RenewalNumber, #VoucherPrefix, #VoucherNumber, #ClientId, #Issue, #ContractStart, #ContractEnd, #EffectDate, #Status, #Notes, #NetAmount,#FeeAmount,#PenaltyAmount, #TotalAmount, #PlateNumber, #BirthDate, #LicenceDate, #DurationID, #BM, #MotorUseID, #CoverPacketID, #RatingAreaID, #PostCode, #MotorValue, #HP, #ConstructionYear, #EndorsementType, #InsuredName, #InsuredAddress, #BrandID, #Brand, #ModelID, #Model, #TypeID, #ColorID, #Color, #HasABS, #HasAirbags, #HasAlarm, #HasImmobilizer, #IsCabrio, #IsLiquidGas, #IsBrandNew, #HasParking, #HasHomeOwned, #AnnualUse, #EmployeeArea, #DriversOptions, #EngineCC, #Weight, #WheelBase, 10)";
if (Conn.State == ConnectionState.Closed)
{
Conn.Open();
}
SqlCommand InsContractCmd = new SqlCommand(insertCmd, Conn);
try
{
InsContractCmd.Parameters.Add("#ID", SqlDbType.Int);
if (contract.ID == null)
{
InsContractCmd.Parameters["#ID"].Value = 0;
}
else
{
InsContractCmd.Parameters["#ID"].Value = contract.ID;
};
InsContractCmd.Parameters.Add("#BranchID", SqlDbType.Int);
if (contract.BranchID == null)
{
InsContractCmd.Parameters["#BranchID"].Value = 0;
}
else
{
InsContractCmd.Parameters["#BranchID"].Value = contract.BranchID;
};
//i do the same thing for all fields on the Inserted record
// and then i Execute the query
RetRes = InsContractCmd.ExecuteNonQuery();
}
catch (Exception e )
{
bool Res = InsertErrors(Conn, "Contract", e);
RetRes = 99;
}
return RetRes;
}
// Code for Inserting Record
After Iterating I get a list on my screen for all the succeded iteration. It reports back to me that all data from JSON where iterated but I get inserted fewer records in my table.
Since I use try catch blocks, I have also created a small procedure to capture error messages and insert them into a database table But the iteration never hits my catch block in the insert procedure.
Any help would be greatly appreciated.
Probably it is due to empty fields check your json all desire fields have data properly exist or not.
Secondly apply R.E for proper data on json for verification.
Hopefully it help you out.
I have a c# script task in an SSIS job that calls an API for the purpose of Geocoding. The API is proprietary and works something like this, receives request, takes address string, then attempts to string match to a huge list of addresses (millions) and if it cannot find it, then go out to another service such as google and get geodata info.
As you can imagine this string matching takes up a lot of time per request. Sometimes it's as slow as one request per min, and I have 4M addresses I need to do this for. Getting any dev work on the API side of things is not an option. To give a better picture of the process here is what I'm doing currently:
I pull a list of addresses from database (about 4M) and put it in a datatable and set variables:
// Fill c# datatable with query results
sdagetGeoData.Fill(dtGeoData);
// check to ensure datable has rows
if (dtGeoData.Rows.Count > 0)
{
// if datatable has rows, for every row set the varible
foreach (System.Data.DataRow row in dtGeoData.Rows)
{
localID = row[0].ToString();
address = row[1].ToString();
city = row[2].ToString();
state = row[3].ToString();
zip = row[4].ToString();
country = row[5].ToString();
// after varaibles are set, now run this method to post, get response and insert the string
GetGLFromAddress();
}
}
GetGLFromAddress() works like this:
Take the variables from above and form the JSON. Send the JSON using "POST" and httpWebRequest. Wait for request (time consuming). Return request. Set new variables with the return. Use those variables to update/ insert back into database, THEN loop through the next row in original datatable.
It's important to understand this flow because I need to be able to keep the localID variable with each request so I can update the correct record in the database.
Here is GetGLFromAddress():
private void GetGLFromAddress()
{
// Request JSON data with Payload
var httpWebRequest = (HttpWebRequest)WebRequest.Create("http:");
httpWebRequest.Headers.Add("Authorization", "");
httpWebRequest.ContentType = "application/json";
httpWebRequest.Method = "POST";
using (var streamWriter = new StreamWriter(httpWebRequest.GetRequestStream()))
{
// this takes the variables from your c# datatable and formats them for json post
var jS = new JavaScriptSerializer();
var newJson = jS.Serialize(new SeriesPost()
{
AddressLine1 = address,
City = city,
StateCode = state,
CountryCode = country,
PostalCode = zip,
CreateSiteIfNotFound = true
});
//// So you can see the JSON thats output
System.Diagnostics.Debug.WriteLine(newJson);
streamWriter.Write(newJson);
streamWriter.Flush();
streamWriter.Close();
}
try
{
var httpResponse = (HttpWebResponse)httpWebRequest.GetResponse();
using (var streamReader = new StreamReader(httpResponse.GetResponseStream()))
{
var result = streamReader.ReadToEnd();
// javascript serializer... deserializing the returned json so that way you can set the variables used for insert string
var p1 = new JavaScriptSerializer();
// after this line, obj is a fully deserialzed string of json Notice how I reference obj[x].fieldnames below. If you ever want to change the fiels or bring more in
// this is how you do it.
var obj = p1.Deserialize<List<RootObject>>(result);
// you must ensure the values returned are not null before trying to set the variable. You can see when that happens, I'm manually setting the variable value to null.
if (string.IsNullOrWhiteSpace(obj[0].MasterSiteId))
{
retGLMID = "null";
}
else
{
retGLMID = obj[0].MasterSiteId.ToString();
}
if (string.IsNullOrWhiteSpace(obj[0].PrecisionName))
{
retAcc = "null";
}
else
{
retAcc = obj[0].PrecisionName.ToString();
}
if (string.IsNullOrWhiteSpace(obj[0].PrimaryAddress.AddressLine1Combined))
{
retAddress = "null";
}
else
{
retAddress = obj[0].PrimaryAddress.AddressLine1Combined.ToString();
}
if (string.IsNullOrWhiteSpace(obj[0].Latitude))
{
retLat = "null";
}
else
{
retLat = obj[0].Latitude.ToString();
}
if (string.IsNullOrWhiteSpace(obj[0].Longitude))
{
retLong = "null";
}
else
{
retLong = obj[0].Longitude.ToString();
}
retNewRecord = obj[0].IsNewRecord.ToString();
// Build insert string... notice how I use the recently created variables
// string insertStr = retGLMID + ", '" + retAcc + "', '" + retAddress + "', '" + retLat + "', '" + retLong + "', '" + localID;
string insertStr = "insert into table " +
"(ID,GLM_ID,NEW_RECORD_IND,ACCURACY) " +
" VALUES " +
"('" + localID + "', '" + retGLMID + "', '" + retNewRecord + "', '" + retAcc + "')";
string connectionString = "Data Source=; Initial Catalog=; Trusted_Connection=Yes";
using (SqlConnection connection = new SqlConnection(connectionString))
{
SqlCommand cmd = new SqlCommand(insertStr);
cmd.CommandText = insertStr;
cmd.CommandType = CommandType.Text;
cmd.Connection = connection;
connection.Open();
cmd.ExecuteNonQuery();
connection.Close();
}
}
}
{
string insertStr2 = "insert into table " +
"(ID,GLM_ID,NEW_RECORD_IND,ACCURACY) " +
" VALUES " +
"('" + localID + "', null, null, 'Not_Found')";
string connectionString2 = "Data Source=; Initial Catalog=; Trusted_Connection=Yes";
using (SqlConnection connection = new SqlConnection(connectionString2))
{
SqlCommand cmd = new SqlCommand(insertStr2);
cmd.CommandText = insertStr2;
cmd.CommandType = CommandType.Text;
cmd.Connection = connection;
connection.Open();
cmd.ExecuteNonQuery();
connection.Close();
}
}
}
When I have attempted to use Parallel.Foreach, I had issues with the variables. I'd like to have multiple requests ran, but to retain each instance of the variable per request if that makes sense. I have no way to pass the localID to the API and return it, or that would be ideal.
Is this even possible?
And how would I need to structure this call to achieve what I am after?
Essentially I want to be able to send multiple calls, to speed up the entire process.
EDIT: added the code for GetGlFromAddress(). Yes, I am a newb, so please be kind :)
Put all your data in an array and you could call more than one request at a time, it is best to use multi tasks or Async Methods to call API.
I'm selecting about 20,000 records from the database and then I update them one by one.
I looked for this error and I saw that setting the CommandTimeout will help, but not in my case.
public void Initialize()
{
MySqlConnectionStringBuilder SQLConnect = new MySqlConnectionStringBuilder();
SQLConnect.Server = SQLServer;
SQLConnect.UserID = SQLUser;
SQLConnect.Password = SQLPassword;
SQLConnect.Database = SQLDatabase;
SQLConnect.Port = SQLPort;
SQLConnection = new MySqlConnection(SQLConnect.ToString());
}
public MySqlDataReader SQL_Query(string query)
{
MySqlCommand sql_command;
sql_command = SQLConnection.CreateCommand();
sql_command.CommandTimeout = int.MaxValue;
sql_command.CommandText = query;
MySqlDataReader query_result = sql_command.ExecuteReader();
return query_result;
}
public void SQL_NonQuery(string query)
{
MySqlCommand sql_command;
sql_command = SQLConnection.CreateCommand();
sql_command.CommandTimeout = int.MaxValue;
sql_command.CommandText = query;
sql_command.ExecuteNonQuery();
}
And here is my method which makes the select query:
public void CleanRecords()
{
SQLActions.Initialize();
SQLActions.SQL_Open();
MySqlDataReader cashData = SQLActions.SQL_Query("SELECT `cash`.`id`, SUM(`cash`.`income_money`) AS `income_money`, `cash_data`.`total` FROM `cash_data` JOIN `cash` ON `cash`.`cash_data_id` = `cash_data`.`id` WHERE `user`='0' AND `cash_data`.`paymentterm_id`='0' OR `cash_data`.`paymentterm_id`='1' GROUP BY `cash_data_id`");
while(cashData.Read()){
if(cashData["income_money"].ToString() == cashData["total"].ToString()){
UpdateRecords(cashData["id"].ToString());
}
}
SQLActions.SQL_Close();
}
And here is the method which makes the update:
public void UpdateRecords(string rowID)
{
SQLActions.Initialize();
SQLActions.SQL_Open();
SQLActions.SQL_NonQuery("UPDATE `cash_data` SET `end_date`='" + GetMeDate() + "', `user`='1' WHERE `id`='" + rowID + "'");
SQLActions.SQL_Close();
}
Changing the database structure is not an option for me.
I thought that setting the timeout to the maxvalue of int will solve my problem, but is looks like this wont work in my case.
Any ideas? :)
EDIT:
The error which I get is "Fatal error encoutered during data read".
UPDATE:
public void CleanRecords()
{
StringBuilder dataForUpdate = new StringBuilder();
string delimiter = "";
SQLActions.Initialize();
SQLActions.SQL_Open();
MySqlDataReader cashData = SQLActions.SQL_Query("SELECT `cash`.`id`, SUM(`cash`.`income_money`) AS `income_money`, `cash_data`.`total` FROM `cash_data` JOIN `cash` ON `cash`.`cash_data_id` = `cash_data`.`id` WHERE `user`='0' AND `cash_data`.`paymentterm_id`='0' OR `cash_data`.`paymentterm_id`='1' GROUP BY `cash_data_id`");
while (cashData.Read())
{
if (cashData["income_money"].ToString() == cashData["total"].ToString())
{
dataForUpdate.Append(delimiter);
dataForUpdate.Append("'" + cashData["id"].ToString() + "'");
delimiter = ",";
}
}
SQLActions.SQL_Close();
UpdateRecords(dataForUpdate.ToString());
}
public void UpdateRecords(string rowID)
{
SQLActions.Initialize();
SQLActions.SQL_Open();
SQLActions.SQL_NonQuery("UPDATE `cash_data` SET `end_date`='" + GetMeDate() + "', `user`='1' WHERE `id` IN (" + rowID + ")");
SQLActions.SQL_Close();
}
You may be able to use
UPDATE cash_data .... WHERE id IN (SELECT ....)
and do everything in one go. Otherwise, you could do it in two steps: first the select collects all the ids, close the connection and then do the update in obne go with all the ids.
The code for the second option might look something like this:
public void CleanRecords()
{
StringBuilder builder = new StringBuilder();
string delimiter = "";
SQLActions.Initialize();
SQLActions.SQL_Open();
MySqlDataReader cashData = SQLActions.SQL_Query("SELECT `cash`.`id`, SUM(`cash`.`income_money`) AS `income_money`, `cash_data`.`total` FROM `cash_data` JOIN `cash` ON `cash`.`cash_data_id` = `cash_data`.`id` WHERE `user`='0' AND `cash_data`.`paymentterm_id`='0' OR `cash_data`.`paymentterm_id`='1' GROUP BY `cash_data_id`");
while(cashData.Read()){
if(cashData["income_money"].ToString() == cashData["total"].ToString()){
builder.Append(delimiter);
builder.Append("'" + cashData["id"].ToString() + "'");
delimiter = ",";
}
}
SQLActions.SQL_Close();
UpdateRecords(builder.ToString());
}
public void UpdateRecords(string rowIDs)
{
SQLActions.Initialize();
SQLActions.SQL_Open();
SQLActions.SQL_NonQuery("UPDATE `cash_data` SET `end_date`='" + GetMeDate() + "', `user`='1' WHERE `id` IN (" + rowIDs + ")";
SQLActions.SQL_Close();
}
There are multiple problem:
First: You have reading information around 20K using data reader and then doing update one by one in reader itself. Reader holds the connection open until you are finished. So this is not the good way to do it. Solution: We can read the information using Data Adapter.
Second: Rather than doing one by one update, we can update in bulk in one go. There are multiple option for bulk operation. In SQL u can do either by sending information in XML format or u can use Table Valued Parameter (TVP) (http://www.codeproject.com/Articles/22205/ADO-NET-and-OPENXML-to-Perform-Bulk-Database-Opera) OR (http://dev.mysql.com/doc/refman/5.5/en/load-xml.html)