Recently I've been tasked with creating an automated ETL process that pumps the data into tables based on the flat file name by reading a master mapping file. I've decided to go with SqlBulkCopy and everything seemed to be fine. IDataReader interface was implemented to read flat-files, SQL Server's meta-data provided with number of columns for a one-to-one data mapping, everything was working until I ran across the file that carried empty strings. SqlBulkCopy throws an Exception saying that "The given value of type String from the data source cannot be converted to type int of the specified target column.". End of story, it does not even care that the DB type for this column is INT NULL. I know that I can interpret meta-data further, extract data types for given columns, build a DataSet based on extracted information, re-cast the data from flat-files and get myself a nice strongly typed solution that will work, but I am a lazy guy who feels like his happiness was viciously lacerated by Microsoft, or my own incompetence if someone knows of a solution to my sudden problem. Thank you for your time.
List<String> fileNames;
DateTime startJobTime = DateTime.Now;
Console.WriteLine("---------------------------------------------");
Console.WriteLine("Start Time: " + startJobTime);
Console.WriteLine("---------------------------------------------");
using (SqlConnection sqlCon = new SqlConnection(sqlConnection))
{
try
{
sqlCon.Open();
sqlCon.ChangeDatabase(edwDBName);
// Get service information for staging job
UnivStage us = GetStagingJobInfo(jobName, sqlCon);
us.StartJobTime = startJobTime;
// Get a list of file names
fileNames = GetFileList(us, args);
if (fileNames.Count > 0)
{
// Truncate Staging Table
TruncateStagingTable(us, sqlCon);
// Close and dispose of sqlCon2 connection
sqlCon.Close();
Console.WriteLine("Processing files: ");
foreach (String fileName in fileNames)
Console.WriteLine(fileName);
Console.WriteLine();
}
else
{
Console.WriteLine("No files to process.");
Environment.Exit(0);
}
// Re-open Sql Connection
sqlCon.Open();
sqlCon.ChangeDatabase(stagingDBName);
foreach (String filePath in fileNames)
{
using (SqlTransaction sqlTran = sqlCon.BeginTransaction())
{
using (FlatFileReader ffReader = new FlatFileReader(filePath, us.Delimiter))
{
using (SqlBulkCopy sqlBulkCopy =
new SqlBulkCopy(sqlCon, SqlBulkCopyOptions.Default, sqlTran))
{
SqlConnection sqlCon2 = new SqlConnection(sqlConnection);
SetColumnList(sqlCon2, us, sqlBulkCopy);
sqlBulkCopy.BatchSize = 1000;
sqlBulkCopy.DestinationTableName =
us.StagingSchemaName + "." + us.StagingTableName;
sqlBulkCopy.WriteToServer(ffReader);
sqlTran.Commit();
sqlCon2.Close();
}
}
}
}
sqlCon.ChangeDatabase(edwDBName);
sqlCon.Close();
sqlCon.Open();
SetRowCount(us, sqlCon);
sqlCon.Close();
us.EndJobTime = DateTime.Now;
sqlCon.Open();
LogStagingProcess(us, sqlCon);
sqlCon.Close();
Console.WriteLine(us.ProcessedRowCount + " rows inserted.");
Console.WriteLine("---------------------------------------------");
Console.WriteLine("Success! End Time: " + us.EndJobTime);
Console.WriteLine("---------------------------------------------");
Console.ReadLine();
}
catch (SqlException e)
{
RenderExceptionMessagesAndExit(e,
"Exception have occured during an attempt to utilize SqlBulkCopy\n");
}
}
Convert your empty strings to DBNull.
Related
I am attempting to query data from within an Excel sheet using my C# application. I seem to be able to connect to the file now but I am getting the following exception...
The Microsoft Office Access database engine could not find the object 'Sheet1$'.
I'm using a connection string similar to that of Excel connection strings
This is my code...
The code is basically a method (tucked away in a class) that uses a parameter variable to return a data from one column based on provided data in another....This same method works for SQL connections in my other applications..this is my first swing at an excel sheet however..
public static string ReturnDefinition(string remCode)
{
string strReturnMessage = "";
string excelConnectString = #"Provider=Microsoft.ACE.OLEDB.12.0;Data Source=MEDICARE.xlsx;Extended Properties=""Excel 12.0 Xml;HDR=YES""";
OleDbConnection objConn = new OleDbConnection(excelConnectString);
OleDbCommand objCmd = new OleDbCommand("Select * from [Sheet1$] where Code = #remCode", objConn);
objCmd.Parameters.AddWithValue("#remCode", remCode);
try
{
objConn.Open();
OleDbDataReader ExcelDataReader = objCmd.ExecuteReader(CommandBehavior.CloseConnection);
if (ExcelDataReader.HasRows)
{
while (ExcelDataReader.Read())
{
if (string.IsNullOrEmpty((string)ExcelDataReader["Description"]))
{
strReturnMessage = "** ERROR **";
}
else
{
strReturnMessage = ExcelDataReader["Description"].ToString();
}
}
}
else
{
strReturnMessage = "** ERROR **";
}
ExcelDataReader.Close();
return strReturnMessage;
}
catch (Exception ex)
{
return "** ERROR **: " + ex.Message;
}
finally
{
}
}
5/30
I realize there is literature out there that covers connections to Excel using OLEDB but I think I've boiled the issue down to a read issue in the sheet its self. Again, this is the first time I've tried to connect to Excel. My HDR is set to true, as I intend to treat the sheet like a SQL table.
Changed Excel to v.14.0 in Extended Properties
I may have been targeting the wrong version of excel. Specifically, the excel sheet was created using Office 2010. Using This Article for reference, I changed the version from 12.0 to 14.0.
Now I get Could not find installable ISAM
Well, I think I've got it figured out for now.
At least I can get to the sheet anyway.
I'll just need to sort out whats going on with my data types.
I now get Data type mismatch in criteria expression.
The data being passed from my parameter variable is a string but the column name ("Code") I am trying to reach contains strings in some rows and int32 in others. If it were SQL I would say this column would be of type CHAR(whatever). I don't know in this case.
Anyway, here is my own resolution...it seems all I needed to do was assign the sheet name to a variable and then include that variable in my query.
enter public static string ReturnDefinition(string remCode)
{
string strReturnMessage = "";
string excelConnectString = #"Provider=Microsoft.ACE.OLEDB.12.0;Data Source=MEDICARE.xlsx;Extended Properties=""Excel 12.0 Xml;HDR=YES""";
string excelSheet = "Sheet1$";
OleDbConnection objConn = new OleDbConnection(excelConnectString);
OleDbCommand excelCmd = new OleDbCommand("Select * from ["+ excelSheet + "] where Code = #remCode", objConn);
excelCmd.Parameters.AddWithValue("#remCode", remCode);
try
{
objConn.Open();
OleDbDataReader ExcelDataReader = excelCmd.ExecuteReader(CommandBehavior.CloseConnection);
if (ExcelDataReader.HasRows)
{
while (ExcelDataReader.Read())
{
if (string.IsNullOrEmpty((string)ExcelDataReader["Description"]))
{
strReturnMessage = "** ERROR **";
}
else
{
strReturnMessage = ExcelDataReader["Description"].ToString();
}
}
}
else
{
strReturnMessage = "** ERROR **";
}
ExcelDataReader.Close();
return strReturnMessage;
}
catch (Exception ex)
{
return "** ERROR **: " + ex.Message;
}
finally
{
}
}
When I connect to the DB and I want to read data from table it doesn't work an shows these
Exception Details:
System.IndexOutOfRangeException: Cannot find table 0.
Error in Line 141
I have only one table in the dataset. Do you know the best way to read from table for me ? I return only one table. When I use foreach, how can I read a one row data?
internal DataSet read(string query)
{
DataSet nds = new DataSet();
try
{
string connectionString = "...";
try
{
SqlConnection nsqlc = new SqlConnection(connectionString);
SqlCommand get = new SqlCommand(query, nsqlc);
nsqlc.Open();
SqlDataAdapter nsda = new SqlDataAdapter(get);
nsda.Fill(nds);
nsqlc.Close();
}
catch (Exception)
{
}
return nds;
}
catch (Exception)
{
return nds;
}
}
Main form :
links = links + t.Tables[0].Rows[i][0].ToString() + "%";
String links = "";
links = "movie%";
DataSet t = n.read("select id , poster from Movie order by id desc");
for (int i = 0; i < 10; i++)
{
links = links + t.Tables[0].Rows[i][0].ToString() + "%";
links = links + t.Tables[0].Rows[i][1] + "%";
}
The closest thing to an answer here would have to be that the n.read method returns a DataSet instance that has no tables in it. You need to step into the method with a debugger and figure out why it behaves like that.
If you see what the problem is, fix it. If not, share the read method code and someone will be able to help you further.
Looking at your updated post, I see that you are swallowing every possible exceptions that could occur while trying to fetch data. Remove the catch block from your read method. This will allow you to see what the real problem is.
Exception swallowing is something you'll want to take away from your code on a global scale. I'd suggest that you do some Googling to find out why it's a terrible habit.
About a "better way to read from table", you should consider using a IDataReader.
string links = "movie%";
using (var connection = new SqlConnection("your connection string");
{
using (var command = someExistingConnection.CreateCommand())
{
command.CommandText = "select id, poster from Movie order by id desc";
command.Connection = connection;
connection.Open();
try
{
using (var reader = command.ExecuteReader())
{
while (reader.Read())
{
var idValue = reader.GetObject(0).ToString(); // would be better to use actual field type, which is unknown at the time I'm writing this
var posterValue = reader.GetString(1);
// perform concatenation here using the two variables declared above
links += String.Format("{0}%{1}%", idValue, posterValue);
}
}
}
finally
{
connection.Close();
}
}
}
This will outperform working with a DataSet by a long shot.
I am trying to print out results from a MySQL query on a Visual C# Console application. I am able to get multiple columns on one line as you can see below, but I am wondering how I can get multiple results (rows). You see, my table holds more records that meet the query criteria. Can someone help me out?
class Program
{
static void Main(string[] args)
{
string ConnectionString = "Server=localhost; Database=world; Uid=root; Pwd=password"; // giving connection string
MySqlConnection connection = new MySqlConnection(ConnectionString);
MySqlCommand cmd = connection.CreateCommand();
cmd.CommandText = "SELECT name, population FROM city where population > 4000000";
try
{
connection.Open();
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
MySqlDataReader reader = cmd.ExecuteReader();
while (reader.Read())
{
Console.WriteLine("City name is: " + reader["name"].ToString() + " " + reader["population"].ToString());
Console.Read();
}
}
Your call to Console.Read() is blocking the while loop, so you're only printing one line to console, and then waiting for user input.
Cheers
You want to remove that Console.Read();, it is blocking your application from continuing on until it reads another character input on the Console.
Other things to consider: Using statements ensure that the unmanaged resources consumed by the MySql objects are released when the objects are no longer in use. Use Parameterized queries (aka Prepared Statements) as they performs better and are more secure.
string sql = "SELECT name, population FROM city WHERE population > #population"
using (var conn = new MySqlConnection(/*Connection String*/))
{
conn.Open();
using (var cmd = new MySqlCommand(sql, conn))
{
cmd.Parameters.AddWithValue("#population", 4000000);
using (var reader = cmd.ExecuteReader())
{
while (reader.Read())
{
Console.WriteLine("City: {0} Population: {1}",
reader["name"], reader["population"]);
}
}
}
}
There may be other ways to do this, but the best way that I know of is loading the datareader into a datatable.
DataTable dt = new DataTable("City");
dt.Fill(reader);
foreach (DataRow row in dt.Rows){
Console.WriteLine("City name is: " + row["name"].ToString() + " " + row["population"].ToString());
Console.Read();
}
EDIT While answering, I realized that the Console.Read() may be the issue. This code will work, but an input to the console will need to be given for each line.
So what I'm doing is reading a lot of data from remote Nettezza database and inserting them into another remote Oracle database. For that I'm using ODBC driver. The problem is that there is a lot of data and it takes too much time. How can I speed up?
Here is what I do:
First I create connection and command for inserting:
String connect = "Driver={Microsoft ODBC for Oracle};CONNECTSTRING=(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=myhost)(PORT=myprt))(CONNECT_DATA=(SERVICE_NAME=myname)));Uid=uid;Pwd=pass";
connection = new OdbcConnection(connect);
connection.Open();
String q = #"INSERT INTO TEST (id)
VALUES (?)";
myCommand = new OdbcCommand(q,connection);
Then I read data from netteza:
String connect = "Driver={NetezzaSQL};servername=server;port=5480;database=db; username=user;password=pass;
string query = #"SELECT T2.LETO_MESEC, T1.*
FROM data T1
JOIN datga2 T2 ON T2.ID = T1.EFT_ID
WHERE T2.LETO_MESEC = '" + mesec + #"'";
using (OdbcConnection connection = new OdbcConnection(connect))
{
try
{
OdbcCommand command = new OdbcCommand(query, connection);
connection.Open();
OdbcDataReader reader = command.ExecuteReader();
int counter=0;
while (reader.Read())
{
int id_first = reader.GetInt32(5);
insertOracle(id_first);
}
}
catch (Exception e)
{
Console.WriteLine("ne dela" + e.ToString());
}
}
And finally my insert :
public void insertOracle(int id_first)
{
try
{
myCommand.Parameters.Clear();
myCommand.Parameters.Add(new OdbcParameter("id", id_first));
myCommand.ExecuteNonQuery();
}
catch (Exception e)
{
Console.WriteLine("ne dela" + e.ToString());
}
}
I noticed that these commit in every line, so how to remove that and speed it up. Right now it takes about 10 minutes for 20000 rows.
Single inserts are always going to be slow -- start processing the data in arrays, selecting a batch of ID's from the source system and loading an array to the target.
Here is an article which might be helpful. http://www.oracle.com/technetwork/issue-archive/2009/09-sep/o59odpnet-085168.html
My application is a winform app that relies on a database. At startup of the application it connects to an SQL Database on the server and put this information in a DataSet/DataTable.
If for some reason the database on the server is not accessible, the application has a built in failover and it will get its information from the local database.
If, in a normal scenario, I start the tool it will read from the sql database and if it has been updated on the server (a seperate snippet checks this), it should make sure the local database is up to date and this is where the problem starts.. (see below)
This part works fine and is added as context - this is where we connect to the SQL Database
public static DataSet dtsTableContents;
public static DataTable CreateDatabaseSQLConnection()
{
try
{
string strSqlConnectionString = "Data Source=MyLocation;Initial Catalog=MyCatalog;User=MyUser;Password=MyPassword;";
SqlCommand scoCommand = new SqlCommand();
scoCommand.Connection = new SqlConnection(strSqlConnectionString);
scoCommand.Connection.Open();
string strQueryToTable = "SELECT * FROM " + strTableName;
dtsTableContents = new DataSet();
SqlCommand scmTableInformation = new SqlCommand(strQueryToTable, scnConnectionToDatabase);
SqlDataAdapter sdaTableInformation = new SqlDataAdapter(scmTableInformation);
scnConnectionToDatabase.Open();
sdaTableInformation.Fill(dtsTableContents, strTableName);
DataTable dttTableInformation = dtsTableContents.Tables[strTableName];
scnConnectionToDatabase.Close();
return dttTableInformation;
}
catch
{
return null;
}
}
This snippet is part of the failover method that reads from my local database...
This part works fine and is added as context - this is where we connect to the MDB Database
public static DataTable CreateDatabaseConnection()
{
try
{
string ConnectionString = #"Provider=Microsoft.Jet.OLEDB.4.0;Data Source=MyLocation;Persist Security Info=True;JET OLEDB:Database Password=MyPassword;"
odcConnection = new OleDbConnection(ConnectionString);
odcConnection.Open();
string strQueryToTable = "SELECT * FROM " + strTableName;
DataSet dtsTableContents = new DataSet();
OleDbCommand ocmTableInformation = new OleDbCommand(strQueryToTable, ocnConnectionToDatabase);
OleDbDataAdapter odaTableInformation = new OleDbDataAdapter(ocmTableInformation);
ocnConnectionToDatabase.Open();
odaTableInformation.Fill(dtsTableContents, strTableName);
DataTable dttTableInformation = dtsTableContents.Tables[strTableName];
ocnConnectionToDatabase.Close();
return dttTableInformation;
}
catch
{
return null;
}
}
From my CreateDatabaseSQLConnection() I have a DataSet. This DataSet is verified to contain all the information from the server database. Now I have been googling around and found myself trying to use this code to update the local database based on this article:
http://msdn.microsoft.com/en-us/library/system.data.common.dataadapter.update(v=vs.71).aspx
public static void UpdateLocalDatabase(string strTableName)
{
try
{
if (CreateDatabaseConnection() != null)
{
string strQueryToTable = "SELECT * FROM " + strTableName;
OleDbDataAdapter odaTableInformation = new OleDbDataAdapter();
odaTableInformation.SelectCommand = new OleDbCommand(strQueryToTable, odcConnection);
OleDbCommandBuilder ocbCommand = new OleDbCommandBuilder(odaTableInformation);
odcConnection.Open();
odaTableInformation.Update(dtsTableContents, strTableName);
odcConnection.Close();
}
}
catch { }
}
This snippet runs error-free but it does not seem to change anything. Also the time it takes to run this step takes like milliseconds and I would think this to take longer.
I am using the DataSet I obtained from my SQL Connection, this DataSet I am trying to write to my local database.
Might it be the fact that this is a DataSet from an SQL connection and that I can't write this to my mdb connection via my OleDbAdapter or am I just missing the obvious here?
Any help is appreciated.
Thanks,
Kevin
For a straight backup from the external database to the internal backup
I've just been messing around with an sdf and a server based sql database and it has worked.. It's not by all means the finished product but for one column I've got the program to read from the external database and then write straight away to the local .sdf in around 15 lines of code
SqlConnection sqlCon = new SqlConnection( ExternalDatabaseConnectionString );
SqlCeConnection sqlCECon = new SqlCeConnection( BackUpConnectionString );
using ( sqlCon )
{
using ( sqlCECon )
{
sqlCon.Open( );
sqlCECon.Open( );
SqlCommand get = new SqlCommand( "Select * from [TableToRead]", sqlCon );
SqlCeCommand save = new SqlCeCommand( "Update [BackUpTable] set InfoColumn = #info where ID = #id", sqlCECon );
SqlDataReader reader = get.ExecuteReader( );
if ( reader.HasRows )
{
reader.Read( );
save.Parameters.AddWithValue("#id", reader.GetString(0));
save.Parameters.AddWithValue( "#info", reader.GetString( 1 ));
save.ExecuteNonQuery( );
}
}
}
For one row of a database, backing up one column, it works, I'm assuming you will have some sort of auto incremented key like ID?
I think a first step would be to reduce your reliance on static methods and fields.
If you look at your UpdateLocalDatabase method you'll see that you are passing in strTableName which is used by that method but a method that UpdateLocalDatabase calls (CreateDatabaseConnection) refers to a different global static variable named the same. Most likely the two strTableName variables contain different values and you are not seeing that they are not the same variable.
Also you are trying to write out the global static data set dtsTableContents in UpdateLocalDatabase but if you then look at CreateDatabaseConnection it actually creates a local version of that variable -- again, you have two variables named the same thing where one is global and one is local.
I suspect that the two variables of dtsTableContents is the problem.
My suggestion, again, would be to not have any static methods or variables and, for what you're doing here, try not to use any global variables. Also, refactor and/or rename your methods to match more what they are actually doing.
After endlessly trying to use the DataAdapter.Update(Method) I gave up.. I settled for using an sdf file instead of an mdb.
If I need to update my database, I remove the local database, create a new one, using the same connectionstring. Then I loop over the tables in my dataset, read the column names and types from it and create tables based on that.
After this I loop over my dataset and insert the contents of my dataset which I filled with the information from the server. Below is the code, this is just 'quick and dirty' as a proof of concept but it works for my scenario.
public static void RemoveAndCreateLocalDb(string strLocalDbLocation)
{
try
{
if (File.Exists(strLocalDbLocation))
{
File.Delete(strLocalDbLocation);
}
SqlCeEngine sceEngine = new SqlCeEngine(#"Data Source= " + strLocalDbLocation + ";Persist Security Info=True;Password=MyPass");
sceEngine.CreateDatabase();
}
catch
{ }
}
public static void UpdateLocalDatabase(String strTableName, DataTable dttTable)
{
try
{
// Opening the Connection
sceConnection = CreateDatabaseSQLCEConnection();
sceConnection.Open();
// Creating tables in sdf file - checking headers and types and adding them to a query
StringBuilder stbSqlGetHeaders = new StringBuilder();
stbSqlGetHeaders.Append("create table " + strTableName + " (");
int z = 0;
foreach (DataColumn col in dttTable.Columns)
{
if (z != 0) stbSqlGetHeaders.Append(", "); ;
String strName = col.ColumnName;
String strType = col.DataType.ToString();
if (strType.Equals("")) throw new ArgumentException("DataType Empty");
if (strType.Equals("System.Int32")) strType = "int";
if (strType.Equals("System.String")) strType = "nvarchar (100)";
if (strType.Equals("System.Boolean")) strType = "nvarchar (15)";
if (strType.Equals("System.DateTime")) strType = "datetime";
if (strType.Equals("System.Byte[]")) strType = "nvarchar (100)";
stbSqlGetHeaders.Append(strName + " " + strType);
z++;
}
stbSqlGetHeaders.Append(" )");
SqlCeCommand sceCreateTableCommand;
string strCreateTableQuery = stbSqlGetHeaders.ToString();
sceCreateTableCommand = new SqlCeCommand(strCreateTableQuery, sceConnection);
sceCreateTableCommand.ExecuteNonQuery();
StringBuilder stbSqlQuery = new StringBuilder();
StringBuilder stbFields = new StringBuilder();
StringBuilder stbParameters = new StringBuilder();
stbSqlQuery.Append("insert into " + strTableName + " (");
foreach (DataColumn col in dttTable.Columns)
{
stbFields.Append(col.ColumnName);
stbParameters.Append("#" + col.ColumnName.ToLower());
if (col.ColumnName != dttTable.Columns[dttTable.Columns.Count - 1].ColumnName)
{
stbFields.Append(", ");
stbParameters.Append(", ");
}
}
stbSqlQuery.Append(stbFields.ToString() + ") ");
stbSqlQuery.Append("values (");
stbSqlQuery.Append(stbParameters.ToString() + ") ");
string strTotalRows = dttTable.Rows.Count.ToString();
foreach (DataRow row in dttTable.Rows)
{
SqlCeCommand sceInsertCommand = new SqlCeCommand(stbSqlQuery.ToString(), sceConnection);
foreach (DataColumn col in dttTable.Columns)
{
if (col.ColumnName.ToLower() == "ssma_timestamp")
{
sceInsertCommand.Parameters.AddWithValue("#" + col.ColumnName.ToLower(), "");
}
else
{
sceInsertCommand.Parameters.AddWithValue("#" + col.ColumnName.ToLower(), row[col.ColumnName]);
}
}
sceInsertCommand.ExecuteNonQuery();
}
}
catch { }
}