I'm using SSIS and am importing multiple (30) txt files. I create the table on the fly using the file name of the txt file and creating the columns based on the first row from the txt file - all this works. My code stops working when an apostrophe ' is in one of the fields.
The files are delimitated using a |
My code:
SqlConnection myADONETConnection = new SqlConnection();
myADONETConnection = (SqlConnection)(Dts.Connections["xxxxxxxxxxxxxx"].AcquireConnection(Dts.Transaction) as SqlConnection);
string line1 = "";
//Reading file names one by one
string SourceDirectory = #"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
string[] fileEntries = Directory.GetFiles(SourceDirectory);
foreach (string fileName in fileEntries)
{
// do something with fileName
string columname = "";
//Reading first line of each file and assign to variable
System.IO.StreamReader file2 =
new System.IO.StreamReader(fileName);
string filenameonly = ((((fileName.Replace(SourceDirectory, "")).Replace(".txt", "")).Replace("\\", "")).Replace("-", "_"));
line1 = (" IF EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo]." + filenameonly + "') AND type in (N'U'))DROP TABLE [dbo]." + filenameonly + " Create Table dbo." + filenameonly + "([" + file2.ReadLine().Replace("|", "] NVARCHAR(500),[") + "] NVARCHAR(500))").Replace(".txt", "");
file2.Close();
SqlCommand myCommand = new SqlCommand(line1, myADONETConnection);
myCommand.ExecuteNonQuery();
MessageBox.Show("TABLE IS CREATED");
//Writing Data of File Into Table
int counter = 0;
string line;
System.IO.StreamReader SourceFile =
new System.IO.StreamReader(fileName);
while ((line = SourceFile.ReadLine()) != null)
{
if (counter == 0)
{
columname = line.ToString();
columname = "[" + columname.Replace("|", "],[") + "]";
}
else
{
string query = "Insert into dbo." + filenameonly + "(" + columname + ") VALUES('" + line.Replace("|", "','") + "')";
SqlCommand myCommand1 = new SqlCommand(query, myADONETConnection);
myCommand1.ExecuteNonQuery();
}
counter++;
}
SourceFile.Close();
The offending line is:
string query = "Insert into dbo." + filenameonly + "(" + columname + ") VALUES('" + line.Replace("|", "','") + "')";
I tried amending to the below to replace the apostrophe to no avail:
string query = "Insert into dbo." + filenameonly + "(" + columname + ") VALUES('" + line.Replace("|", "','") + line.Replace("'''", "") + "')";
string query = "Insert into dbo." + filenameonly + "(" + columname + ") VALUES('" + line.Replace("'", "") + "')";
Also nesting the replace does not work:
string query = "Insert into dbo." + filenameonly + "(" + columname + ") VALUES('" + line.Replace(line.Replace("'''", ""),"|" ) + "')";
The below does not import anything:
string query = "Insert into dbo." + filenameonly + "(" + columname + ") VALUES('" + line.Replace("|", "','").Replace("'", "") + "')";
Changing to the below imports all but then fails over at the line with the apostrophe:
string query = "Insert into dbo." + filenameonly + "(" + columname + ") VALUES('" + line.Replace("|", "','").Replace("'''", "") + "')";
What am I missing?
Instead of concatenating the two REPLACE() functions like your first attempt, you should nest them.
Replace(
Replace({arguments to remove apostrophe character}),
{arguments to remove pipe character}
)
If you want to keep the C# usage of string.Replace(), you would nest like this:
line.Replace({arguments to replace apostrophe).Replace({arguments to replace pipe})
Or you could do it in two separate statements:
line = line.Replace({arguments to replace apostrophe});
line = line.Replace({arguments to replace pipe});
Related
Requesting your help on below error.
Error: ExecuteNonQuery: Connection property has not been initialized.
I am trying to connect with snowflake DB using script task of SSIS using CSharp code. I have created the connection successfully in connection manager in SSIS and using same connection in C-Sharp code to perform some SQL operation on Snowflake DB. but getting above error in below line.
CreateTableCmd.ExecuteNonQuery();--- error coming on this code
Can somebody please help as I am new to C-Sharp. Also same code is working when change connection of Snowflake to SqlServer.
public void Main() {
string FolderPath = Dts.Variables["User::FolderPath"].Value.ToString();
string TableName = Dts.Variables["User::TableName"].Value.ToString();
string SchemaName = Dts.Variables["User::SchemaName"].Value.ToString();
// string StartingColumn = Dts.Variables["User::StartingColumn"].Value.ToString();
// string EndingColumn = Dts.Variables["User::EndingColumn"].Value.ToString();
// string StartReadingFromRow = Dts.Variables["User::StartReadingFromRow"].Value.ToString();
var directory = new DirectoryInfo(FolderPath);
FileInfo[] files = directory.GetFiles();
//Declare and initilize variables
string fileFullPath = "";
//Get one Book(Excel file at a time)
foreach (FileInfo file in files)
{
fileFullPath = FolderPath + "\\" + file.Name;
//Create Excel Connection
string ConStr;
string HDR;
HDR = "YES";
ConStr = "Provider=Microsoft.ACE.OLEDB.12.0;Data Source=" + fileFullPath + ";Extended Properties=\"Excel 12.0;HDR=" + HDR + ";IMEX=0\"";
OleDbConnection cnn = new OleDbConnection(ConStr);
//MessageBox.Show(TableName);
//Get Sheet Name
cnn.Open();
DataTable dtSheet = cnn.GetOleDbSchemaTable(OleDbSchemaGuid.Tables, null);
//string sheetname;
//sheetname = "";
//foreach (DataRow drSheet in dtSheet.Rows)
//{
//if (drSheet["TABLE_NAME"].ToString().Contains("$"))
//{
// sheetname = drSheet["TABLE_NAME"].ToString();
//Load the DataTable with Sheet Data so we can get the column header
//OleDbCommand oconn = new OleDbCommand("select top 1 * from [" + sheetname + StartingColumn + StartReadingFromRow + ":" + EndingColumn + "]", cnn);
OleDbCommand oconn = new OleDbCommand("select top 1 * from [Part_D_Cut_Points$]", cnn);
OleDbDataAdapter adp = new OleDbDataAdapter(oconn);
DataTable dt = new DataTable();
adp.Fill(dt);
cnn.Close();
int ColumnCount = dt.Columns.Count;
//Prepare Header columns list so we can run against Database to get matching columns for a table.
string ExcelHeaderColumn = "";
String SelectColumn = "";
//string GetHeader ColumnsList to Create Table= "";
for (int i = 0; i < dt.Columns.Count; i++)
{
String check = dt.Columns[i].ColumnName;
if (i != dt.Columns.Count - 1)
ExcelHeaderColumn += "[" + dt.Columns[i].ColumnName + "] NVarchar(150) NULL" + ",";
else
ExcelHeaderColumn += "[" + dt.Columns[i].ColumnName + "] NVarchar(150) NULL";
}
//string GetSelect ColumnsList Fetch data from ExcelTable "";
for (int i = 0; i < dt.Columns.Count; i++)
{
if (i != dt.Columns.Count - 1)
SelectColumn += "[" + dt.Columns[i].ColumnName + "]" + ",";
else
SelectColumn += "[" + dt.Columns[i].ColumnName + "]";
}
SqlConnection myADONETConnection = new SqlConnection();
myADONETConnection = (SqlConnection)(Dts.Connections["SnowFlakeDB"].AcquireConnection(Dts.Transaction) as SqlConnection);
TableName = "STG_PART_D_CUT_POINTS_RAW";
string CreateTableStatement = "IF EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[" + SchemaName + "].";
CreateTableStatement += "[" + TableName + "]')";
CreateTableStatement += " AND type in (N'U'))DROP TABLE [" + SchemaName + "].";
CreateTableStatement += "[" + TableName + "] Create Table " + SchemaName + ".[" + TableName + "]";
//CreateTableStatement += "([" + line.Replace(FileDelimiter, "] " + ColumnsDataType + ",[") + "] " + ColumnsDataType + ")";
CreateTableStatement += "(ID int IDENTITY(1,1),[FileName] Varchar(70) Null," + ExcelHeaderColumn + ")";
SqlCommand sqlCommand = new SqlCommand(CreateTableStatement, myADONETConnection);
SqlCommand CreateTableCmd = sqlCommand;
CreateTableCmd.ExecuteNonQuery();
//MessageBox.Show(CreateTableStatement);
//Use Columns to get data from Excel Sheet
OleDbConnection cnn1 = new OleDbConnection(ConStr);
cnn1.Open();
//OleDbCommand oconn1 = new OleDbCommand("select"+ file.Name +" AS [FileName]," + SelectColumn + " from [" + sheetname + StartingColumn + StartReadingFromRow + ":" + EndingColumn + "]", cnn1);
OleDbCommand oconn1 = new OleDbCommand("select '" + file.Name + "' AS [FileName]," + SelectColumn + " from [Part_D_Cut_Points$]", cnn1);
OleDbDataAdapter adp1 = new OleDbDataAdapter(oconn1);
DataTable dt1 = new DataTable();
adp1.Fill(dt1);
cnn1.Close();
//Load Data from DataTable to SQL Server Table.
using (SqlBulkCopy BC = new SqlBulkCopy(myADONETConnection))
{
BC.DestinationTableName = SchemaName + "." + TableName;
foreach (var column in dt1.Columns)
BC.ColumnMappings.Add(column.ToString(), column.ToString());
BC.WriteToServer(dt1);
}
//}
//}
}
}
The IF EXISTS (SELECT * FROM sys.objects... statement is not valid SQL in Snowflake. Closest equivalent would be CREATE OR REPLACE TABLE...
I have been searching around and I am either confusing myself or not searching for the right thing.
I have this data reader that pulls some information for a store procedure.. but I don't think I am doing it right.
string constr = ConfigurationManager.ConnectionStrings["PAYROLL"].ConnectionString;
using (SqlConnection con = new SqlConnection(constr))
{
using (SqlCommand cmd = new SqlCommand("DLI_EMPLOYEE_PORTAL_EMPLOYEE_INFORMATION"))
{
cmd.CommandType = CommandType.StoredProcedure;
cmd.Parameters.AddWithValue("#EID", Session["sessionEMPID"].ToString());
cmd.Connection = con;
con.Open();
SqlDataReader dataReader = cmd.ExecuteReader(CommandBehavior.CloseConnection);
while (dataReader.Read())
{
string EMP_FIRST = dataReader["FIRST_NAME"].ToString();
string EMP_LAST = dataReader["LAST_NAME"].ToString();
string EMP_DEPT = dataReader["DEPT"].ToString();
string EMP_DEPT_ID = dataReader["DEPT_ID"].ToString();
body = body + "<p>SUBMITTED BY : (" + Session["sessionEMPID"].ToString() + ") " + EMP_FIRST + " " + EMP_LAST + " - DEPT : " + EMP_DEPT + "</p> " + System.Environment.NewLine;
}
con.Close();
}
}
I just need to query one row based of an employee ID.. and I would rather do it not by stored procedure but a select query.
SELECT e.FIRST_NAME, e.LAST_NAME, e.DEPT_ID, d.NAME
FROM EMPLOYEE AS e
INNER JOIN DEPARTMENT AS d ON e.DEPT_ID = d.ID
WHERE (e.ID = 'sim01')
I am building an HTML body string so that is why I need the information.
body = body + "<p>SUBMITTED BY : (" + Session["sessionEMPID"].ToString() + ") " + EMP_FIRST + " " + EMP_LAST + " - DEPT : " + EMP_DEPT + "</p> " + System.Environment.NewLine;
Any help is greatly appreciated. Thank you.
If all you want to do is use a query instead of a stored procedure, just pass your SQL statement to the Command and set your CommandType to Text. If you only ever expect one row, use if (dataReader.Read() instead of while (dataReader.Read()).
string constr = ConfigurationManager.ConnectionStrings["PAYROLL"].ConnectionString;
using (SqlConnection con = new SqlConnection(constr))
{
using (SqlCommand cmd = new SqlCommand(
"SELECT e.FIRST_NAME, e.LAST_NAME, e.DEPT_ID, d.NAME " +
"FROM EMPLOYEE AS e " +
"INNER JOIN DEPARTMENT AS d ON e.DEPT_ID = d.ID " +
"WHERE (e.ID = #EID)"));
{
cmd.CommandType = CommandType.Text;
cmd.Parameters.AddWithValue("#EID", Session["sessionEMPID"].ToString());
cmd.Connection = con;
con.Open();
SqlDataReader dataReader = cmd.ExecuteReader(CommandBehavior.CloseConnection);
if (dataReader.Read())
{
string EMP_FIRST = dataReader["FIRST_NAME"].ToString();
string EMP_LAST = dataReader["LAST_NAME"].ToString();
string EMP_DEPT = dataReader["DEPT"].ToString();
string EMP_DEPT_ID = dataReader["DEPT_ID"].ToString();
body = body + "<p>SUBMITTED BY : (" + Session["sessionEMPID"].ToString() + ") " + EMP_FIRST + " " + EMP_LAST + " - DEPT : " + EMP_DEPT + "</p> " + System.Environment.NewLine;
}
con.Close();
}
}
If the query can return more than one row, you can add TOP 1 to the query with an ORDER BY <some other field> to grab only the most relevant one.
It us better to use query instead of stored procedure if there is no TSQL logic
I am working on an ssis integration and wrote a script task in c# for automating my imports from csv file to my DBs. It works great but I need help with removing commas from strings in pipes(|aaa,aaa| or |a|,|a|) in the csv. For example "Address, city wide". I want a function that can remove that comma(,). I would paste a snippet of my code and what I have done so far.
#
region Namespaces
using System;
using System.Data;
using Microsoft.SqlServer.Dts.Runtime;
using System.Windows.Forms;
using System.Data.SqlClient;
using System.IO;
//using CsvHelper.Configuration;
#
endregion
namespace ST_7ce5ad6fbc104157b534f4eb484a4417 {
[Microsoft.SqlServer.Dts.Tasks.ScriptTask.SSISScriptTaskEntryPointAttribute]
public partial class ScriptMain: Microsoft.SqlServer.Dts.Tasks.ScriptTask.VSTARTScriptObjectModelBase {
public void Main() {
string datetime = DateTime.Now.ToString("yyyyMMddHHmmss");
try {
//Declare Variables
string SourceFolderPath = Dts.Variables["User::SourceFolder"].Value.ToString();
string FileExtension = Dts.Variables["User::FileExtension"].Value.ToString();
string FileDelimiter = Dts.Variables["User::FileDelimiter"].Value.ToString();
string ArchiveFolder = Dts.Variables["User::ArchiveFolder"].Value.ToString();
string ColumnsDataType = Dts.Variables["User::ColumnsDataType"].Value.ToString();
string SchemaName = Dts.Variables["User::SchemaName"].Value.ToString();
SqlConnection myADONETConnection = new SqlConnection();
myADONETConnection = (SqlConnection)
(Dts.Connections["moviesdb"].AcquireConnection(Dts.Transaction) as SqlConnection);
//Reading file names one by one
string[] fileEntries = Directory.GetFiles(SourceFolderPath, "*" + FileExtension);
foreach(string fileName in fileEntries) {
//Writing Data of File Into Table
string TableName = "";
int counter = 0;
string line;
string ColumnList = "";
//MessageBox.Show(fileName);
System.IO.StreamReader SourceFile =
new System.IO.StreamReader(fileName);
while ((line = SourceFile.ReadLine()) != null) {
if (counter == 0) {
ColumnList = "[" + line.Replace("\"", "").Replace(FileDelimiter, "],[") + "]";
//MessageBox.Show(ColumnList);
//"[" + line.Replace(FileDelimiter, "],[") + "]";
TableName = (((fileName.Replace(SourceFolderPath, "")).Replace(FileExtension, "")).Replace("\\", ""));
string CreateTableStatement = "IF EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[" + SchemaName + "].";
CreateTableStatement += "[" + TableName + "]')";
CreateTableStatement += " AND type in (N'U'))DROP TABLE [" + SchemaName + "].";
CreateTableStatement += "[" + TableName + "] Create Table " + SchemaName + ".[" + TableName + "]";
CreateTableStatement += "([" + line.Replace("\"", "").Replace(FileDelimiter, "] " + ColumnsDataType + ",[") + "] " + ColumnsDataType + ")";
SqlCommand CreateTableCmd = new SqlCommand(CreateTableStatement, myADONETConnection);
CreateTableCmd.ExecuteNonQuery();
//MessageBox.Show(CreateTableStatement);
} else {
string query = "Insert into " + SchemaName + ".[" + TableName + "] (" + ColumnList + ") ";
//query += "VALUES('" + line.Replace(FileDelimiter, "','").Replace("\"", "") + "')";
//query += "VALUES('" + line.Replace(FileDelimiter, "','").Replace("\"", "").Replace("\"'\"", "") + "')";
query += "VALUES('" + line.Replace("'", "").Replace(FileDelimiter, "','").Replace("\"", "") + "')";
// MessageBox.Show(query.ToString());
SqlCommand myCommand1 = new SqlCommand(query, myADONETConnection);
myCommand1.ExecuteNonQuery();
}
counter++;
}
SourceFile.Close();
//move the file to archive folder after adding datetime to it
File.Move(fileName, ArchiveFolder + "\\" + (fileName.Replace(SourceFolderPath, "")).Replace(FileExtension, "") + "_" + datetime + FileExtension);
Dts.TaskResult = (int) ScriptResults.Success;
}
} catch (Exception exception) {
// Create Log File for Errors
using(StreamWriter sw = File.CreateText(Dts.Variables["User::LogFolder"].Value.ToString() +
"\\" + "ErrorLog_" + datetime + ".log")) {
sw.WriteLine(exception.ToString());
Dts.TaskResult = (int) ScriptResults.Failure;
}
}
}#
region ScriptResults declaration
enum ScriptResults {
Success = Microsoft.SqlServer.Dts.Runtime.DTSExecResult.Success,
Failure = Microsoft.SqlServer.Dts.Runtime.DTSExecResult.Failure
};#
endregion
}
}
It actually does what I want and imports successfully, but I need further assistance in tweaking the code to replace commas between strings in pipes in the csv file.
public DataTable InsertToIncludeandReturnErrorTable(DataTable MappingTable, DataTable InsertTable, string TableName)
{
//split data and insert data to datatable and validation
var CS = Serenity.Data.SqlConnections.GetConnectionString("Northwind");
String MyConString = CS.ConnectionString;
SqlConnection con = new SqlConnection();
con.ConnectionString = MyConString;
DataTable returnDataTable = InsertTable.Clone();
con.Open();
foreach (DataRow InsertRow in InsertTable.Rows)
{
try
{
string InsertDBFileld = "";
string DatarowField = "";
foreach (DataRow row in MappingTable.Rows)
{
if (InsertDBFileld == "")
InsertDBFileld = InsertDBFileld + row["TableColumn"].ToString().Replace("\r\n", "");
else
InsertDBFileld = InsertDBFileld + "," + row["TableColumn"].ToString().Replace("\r\n", "");
if (DatarowField == "")
DatarowField = "'" + DatarowField + InsertRow[row["ExcelColumn"].ToString().Replace("\r\n", "")].ToString() + "'";
else
DatarowField = DatarowField + ",'" + InsertRow[row["ExcelColumn"].ToString().Replace("\r\n", "")].ToString() + "'";
}
InsertDBFileld = InsertDBFileld + #",CreatedBy,CreatedDate,ModifiedBy,ModifiedDate";
DatarowField = DatarowField + ",'" + User.Identity.Name + "'," + "'" + DateTime.Now + "'," + "'" + User.Identity.Name + "'," + "'" + DateTime.Now + "'";
using (SqlCommand cmd = new SqlCommand(#"INSERT INTO dbo." + TableName + #"(
" + InsertDBFileld + #"
) VALUES(" + DatarowField + ")", con))
{
cmd.ExecuteNonQuery();
}
}
catch (Exception ex)
{
DataRow returnRow = InsertRow;
returnDataTable.Rows.Add(InsertRow.ItemArray);
}
}
if (con.State == System.Data.ConnectionState.Open)
con.Close();
return returnDataTable;
}
[HttpGet]
public FileContentResult DownLoadFile(string destFilePath)
{
//Generate Excel file with data
destFilePath = destFilePath.Replace("%5c", "\\").Replace("%3a", ":");
byte[] fileBytes = System.IO.File.ReadAllBytes(destFilePath);
string fileName = "ErrorList.xlsx";
return File(fileBytes, System.Net.Mime.MediaTypeNames.Application.Octet, fileName);
}
Orginal Code can detect the column data length and output wrong data row.
How can I get the DataType and Size of a column?
If I upload a excel exceed the data length, output a excel file and fill in the red color in the wrong data cell.
You can check the data type and maximum length from the columns of your DataTable:
Type columnType = InsertTable.Columns["TableColumn"].DataType;
int maxLength = InsertTable.Columns["TableColumn"].MaxLength;
If your table does not include schema information (which I doubt), you can get the schema first from the database with a SqlDataAdapter. The FillSchema method is what you need.
I am trying to connect to access 2010 database using the following string connection. But, it wont make any changes in the database.
OleDbConnection conn = new OleDbConnection("Provider=Microsoft.ACE.Oledb.12.0;Data Source=C:\\Program Files\\LogEntry\\LogEntry.accdb; Persist Security Info = False;");
conn.Open();
String text2send = "INSERT INTO TLC(Name,Department,Position,VisitDate,InTime,OutTime,Purpose,HelpedBy,Campus,HelpCode) VALUES(" + name + "," + department + "," + position + "," + date + "," + hourIn + "," + hourOut + "," + purpose + "," + helpedBy + "," + campus + "," + helpcode + ");";
OleDbCommand cmd = new OleDbCommand(text2send,conn);
conn.Close();
Edit:
This is the edited code that I used with Parameter query.
String name = nameTextbox.Text;
String department = departmentCBox.Text;
String purpose = purposeTextbox.Text;
String position = positionCBox.Text;
String date = inDate.Value.ToString("MM/dd/yyyy");
String helpCode = helpCodeCBox.Text;
String hourOut = ""+OutHour.Text+":"+OutMin+" "+OutMeredian;
String helpedBy= "";
String campus= "";
String helpcode= "";
String hourIn = "" + DateTime.Now.ToString("hh") + ":" +
DateTime.Now.ToString("mm") + " " + DateTime.Now.ToString("tt");
OleDbConnection conn = new OleDbConnection("Provider=Microsoft.ACE.Oledb.12.0;Data Source=C:\\Program Files\\LogEntry\\LogEntry.accdb; Persist Security Info = False;");
conn.Open();
String text2send = "Insert Into TLC([Name],[Department],[Position],[VisitDate],[InTime],[OutTime],[Purpose],[HelpedBy],[Campus],[HelpCode]) VALUE(?,?,?,?,?,?,?,?,?,?);";
OleDbCommand cmd = new OleDbCommand(text2send,conn);
cmd.Parameters.AddWithValue("Name", name);
cmd.Parameters.AddWithValue("Department", department);
cmd.Parameters.AddWithValue("Position", position);
cmd.Parameters.AddWithValue("Purpose", purpose);
cmd.Parameters.AddWithValue("HelpedBy", helpedBy);
cmd.Parameters.AddWithValue("Campus", campus);
cmd.Parameters.AddWithValue("HelpCode", helpcode);
cmd.ExecuteNonQuery();
conn.Close();
add cmd.ExecuteNonQuery(); after your command is created and before you close the connection