I have a program to parse a CSV file from local filesystem to a specified SQL Server table.
Now when i execute the program i get error :
System.IndexOutOfRangeException: 'Cannot find column 1' exception on the line where i the program attempts to populate the datatable.
On closer inspection the error shows that its emanating from row number 3 as shown on this link :
CSV_ERROR
This is how i am reading and saving the CSV file :
static void Main(string[] args)
{
var absPath = #"C:\Users\user\Documents\Projects\MastercardSurveillance\fbc_mc_all_cards.csv";
ProcessFile();
void ProcessFile()
{
string realPath = #"C:\Users\user\Documents\CSV";
string appLog = "CSVERRORS";
var logPath = realPath + Convert.ToString(appLog) + DateTime.Today.ToString("dd -MM-yy") + ".txt";
if (!File.Exists(logPath))
{
File.Create(logPath).Dispose();
}
var dt = GetDATATable();
if (dt == null)
{
return;
}
if (dt.Rows.Count == 0)
{
using (StreamWriter sw = File.AppendText(logPath))
{
sw.WriteLine("No rows imported after reading file " + absPath);
sw.Flush();
sw.Close();
}
return;
}
ClearData();
InsertDATA();
}
DataTable GetDATATable()
{
var FilePath = absPath;
string TableName = "Cards";
string realPath = #"C:\Users\user\Documents\CSV";
string appLog = "CSVERRORS";
var logPath = realPath + Convert.ToString(appLog) + DateTime.Today.ToString("dd -MM-yy") + ".txt";
if (!File.Exists(logPath))
{
File.Create(logPath).Dispose();
}
var dt = new DataTable(TableName);
using (var csvReader = new TextFieldParser(FilePath))
{
csvReader.SetDelimiters(new string[] { "," });
csvReader.HasFieldsEnclosedInQuotes = true;
var readFields = csvReader.ReadFields();
if (readFields == null)
{
using (StreamWriter sw = File.AppendText(logPath))
{
sw.WriteLine("Could not read header fields for file " + FilePath);
sw.Flush();
sw.Close();
}
return null;
}
foreach (var dataColumn in readFields.Select(column => new DataColumn(column, typeof(string)) { AllowDBNull = true, DefaultValue = string.Empty }))
{
dt.Columns.Add(dataColumn);
}
while (!csvReader.EndOfData)
{
var data = csvReader.ReadFields();
if (data == null)
{
using (StreamWriter sw = File.AppendText(logPath))
{
sw.WriteLine(string.Format("Could not read fields on line {0} for file {1}", csvReader.LineNumber, FilePath));
sw.Flush();
sw.Close();
}
continue;
}
var dr = dt.NewRow();
for (var i = 0; i < data.Length; i++)
{
if (!string.IsNullOrEmpty(data[i]))
{
dr[i] = data[i];
}
}
dt.Rows.Add(dr);
}
}
return dt;
}
void ClearData()
{
string SqlSvrConn = #"Server=XXXXXX-5QFK4BL\MSDEVOPS;Database=McardSurveillance;Trusted_Connection=True;MultipleActiveResultSets=true;";
using (var sqlConnection = new SqlConnection(SqlSvrConn))
{
sqlConnection.Open();
// Truncate the live table
using (var sqlCommand = new SqlCommand(_truncateLiveTableCommandText, sqlConnection))
{
sqlCommand.ExecuteNonQuery();
}
}
}
void InsertDATA()
{
string SqlSvrConn = #"Server=XXXXXX-5QFK4BL\MSDEVOPS;Database=McardSurveillance;Trusted_Connection=True;MultipleActiveResultSets=true;";
DataTable table = GetDATATable();
using (var sqlBulkCopy = new SqlBulkCopy(SqlSvrConn))
{
sqlBulkCopy.DestinationTableName = "dbo.Cards";
for (var count = 0; count < table.Columns.Count; count++)
{
sqlBulkCopy.ColumnMappings.Add(count, count);
}
sqlBulkCopy.WriteToServer(table);
}
}
}
How can i identify and possibly exclude the extra data columns being returned from the CSV file?
It appears there is a mismatch between number of columns in datatable and number of columns being read from the CSV file.
Im not sure however how i can account for this with my logic. For now i did not want to switch to using a CSV parse package but rather i need insight on how i can remove the extra column or rather ensure that the splitting takes account of all possible dubious characters.
For clarity i have a copy of the CSV file here :
CSV_FILE
I'm trying to pull the info in a column in excel and show it on a Checkbox List in Windows forms.
Right now I have a list of application names in an excel sheet, I am trying to put the cell values into a string array and attach it to the checklist box.
This is my forms class which handles the windows form:
public Form1()
{
InitializeComponent();
//FilesList filesList = new FilesList();
//AppList testApp = new AppList();
//filesList.DirSearch(#"C:\Users\dbell\Downloads\");
Excel e = new Excel(#"SupportedApps.xlsx", 1);
String[] list = e.ReadApplication();
try
{
checkedListBox1.Items.AddRange(list);
}
catch (ArgumentNullException F)
{
Console.WriteLine("Error: " + F.ToString());
}
}
And this below is my poor attempt to create a method which returns a string array from my worksheet:
public string[] ReadApplication()
{
int column = 0;
int row = 1;
int stringNum = 0;
string[] result = null;
try
{
while (ws.Cells[row, column].Value2 != null)
{
result[stringNum] = ws.Cells[row, column].Value2;
row++;
stringNum++;
}
}
catch(NullReferenceException e)
{
Console.WriteLine("Error: " + e.ToString());
}
return result;
}
At the moment I keep getting null results. I have been able to get this working as a CSV file, however I would like to work with only one excel sheet.
Thanks in advance
Try to use https://www.nuget.org/packages/ClosedXML/ library.
ClosedXML.Excel.IXLWorkbook workbook = new XLWorkbook(#"D:\Test.xlsx");
var worksheet = workbook.Worksheets.First();
int column = 1;
int row = 1;
int stringNum = 0;
List<string> result = new List<string>();
try
{
while (worksheet.Cell(row, column).Value != null && row < worksheet.RowCount())
{
result.Add(worksheet.Cell(row, column).Value.ToString());
row++;
stringNum++;
}
}
catch (NullReferenceException e)
{
Console.WriteLine("Error: " + e.ToString());
}
I have a txt file:
LoginId; No_Intervenant
EF2KBT0; 1003820030
ENHD0KE; 1003820129
E9PM7EP; 1003820153
EFT10OO; 1003820218
I need to create another txt file, that contains an sql UPDATE script from this information like:
UPDATE Contact
Set
Contact.No_Intervenant = '1003820030'
where
ISNULL (Contact.LoginId, '') = 'ER7OZXZ';
I only got this result using a Stringbuilder method, but performing hardcode. What I would like is for the header to be added automatically.
public Form1()
{
InitializeComponent();
}
private static void AddSqlCommand(StringBuilder sql, string[] columns, string[] types, string[] values)
{
sql.AppendLine("UPDATE Contact");
sql.AppendLine("SET");
//skip LoginId columns
for (int i = 1; i < columns.Length; i++)
{
switch (types[i].Trim())
{
case "int":
sql.Append($" Contact.{columns[i].Trim()} = {values[i]}");
//sql.Append($" Contact.{columns[0].TrimStart() } = {values[i]}");
break;
default:
sql.Append($" Contact.No_Intervenant = '{values[i]}'");
break;
}
if (columns.Length > 1 && i != columns.Length - 1)
{
sql.Append(",");
}
sql.AppendLine();
}
sql.AppendLine("WHERE");
sql.AppendLine($" ISNULL(Contact.LoginId, '') = '{values[0]}';");
sql.AppendLine();
}
private static StringBuilder GenerateSqlScript(string[] fileContent)
{
var sqlCommand = new StringBuilder();
string[] types = fileContent[0].Split(';');
string[] columns = fileContent[1].Split(';');
//skip the first line (header)
for (int i = 2; i < fileContent.Length; i++)
{
string[] values = fileContent[i].Split(';');
if (values.Length >= 1)
{
AddSqlCommand(sqlCommand, columns, types, values);
}
}
return sqlCommand;
}
How could I get and Add the header automaticaly? Because I'll probably have to do this for longer files, with more columns and more Update lines for other files, and I would not like to hardcode all the headers of the files, like the example I'll have to do next:
Header:
No_Intervenant;First_Name;Last_Name;Role_SE;EMail;Phone;Extension;Statut;Address_1;Address_2;Zip;CPF;Inscription_Particulier;DHM_Stat_Part;Date_via_ClicSeQur;Last_Update;
Data:
1003820030;NOEL;SANTOS;Particulier;;;;Actif;1528 STREET;VAL-D''OR CA;AAA 5T9;123456789;Actif;;2016-07-19 09:49:43;2019-02-08 14:24:19;
I believe you only need a couple of simple changes to your string interpolation, see below. If you have a update that affects multiple tables you should append your table name to the column in the array.
Here is how I tested, according to your logic the first row of the file should contain your data types (the example you posted doesn't). So either your logic is wrong or the data sample is wrong. It works with the test code.
private void button4_Click(object sender, EventArgs e)
{
var line = new List<string>();
line.Add("string;string");
line.Add("LoginId; No_Intervenant");
line.Add("EF2KBT0; 1003820030");
line.Add("ENHD0KE; 1003820129");
line.Add("E9PM7EP; 1003820153");
line.Add("EFT10OO; 1003820218");
var fileContent = line.ToArray();
var sqlCommand = new StringBuilder();
string[] types = fileContent[0].Split(';');
string[] columns = fileContent[1].Split(';');
//skip the first line (header)
for (int i = 2; i < fileContent.Length; i++)
{
string[] values = fileContent[i].Split(';');
if (values.Length >= 1)
{
AddSqlCommand(sqlCommand, columns, types, values, "client");
}
}
}
Edited Fuction:
private static void AddSqlCommand(StringBuilder sql, string[] columns, string[] types, string[] values, string table)
{
sql.AppendLine($"UPDATE {table}");
sql.AppendLine("SET");
//skip LoginId columns
for (int i = 1; i < columns.Length; i++)
{
switch (types[i].Trim())
{
case "int":
sql.Append($" {columns[i].Trim()} = {values[i]}");
break;
default:
sql.Append($" {columns[i].Trim()} = '{values[i]}'");
break;
}
if (columns.Length > 1 && i != columns.Length - 1)
{
sql.Append(",");
}
sql.AppendLine();
}
sql.AppendLine("WHERE");
sql.AppendLine($" ISNULL({columns[0].Trim()}, '') = '{values[0]}';");
sql.AppendLine();
}
}
I believe in this case the 'MERGE' will be a perfect solution.
It could be something like:
-- HEADER --
MERGE [your table] as trg
USING (VALUES
-- DATA FROM THE FILE --
(id, intervenant),
(id, intervenant)
-- FOOTER
) as src(id, intervenant)
ON [your logic from the WHERE statement]
WHEN MATCHED UPDATE SET
trg.[your column] = src.[your column];
The data from the source file can be loaded into a DataTable object, with UPDATE statements then constructed from this. The header names from the file are obtained from the Columns property of the DataTable, then used to specify the columns used in the UPDATE script. In the example below, additional lines and the GO separator are added in the script for formatting. These aren't essential and can be removed if you prefer to.
using System.Linq;
using System.Data;
using System.IO;
using System.Text;
//get source file
string fullFileName = #"C:\Input Folder\SourceFile.txt";
DataTable dt = new DataTable();
StringBuilder sb = new StringBuilder();
//output .sql script
string sqlScript = #"C:\Output Folder\UpdateScript.SQL";
using (StreamReader sr = new StreamReader(fullFileName))
{
string firstLine = sr.ReadLine();
string[] headers = firstLine.Split(';');
//define columns for data table
foreach (string h in headers)
{
dt.Columns.Add(h);
}
int columnCount = dt.Columns.Count;
string line = sr.ReadLine();
while (line != null)
{
string[] fields = line.Split(';');
int currentLength = fields.Count();
if (currentLength < columnCount)
{
while (currentLength < columnCount)
{
line += sr.ReadLine();
currentLength = line.Split(';').Count();
}
fields = line.Split(';');
}
//load data table
dt.Rows.Add(fields);
line = sr.ReadLine();
}
foreach (DataRow dr in dt.Rows)
{
sb.AppendLine("UPDATE Contact SET " + dt.Columns[1] + " = '" + dr[1] +
"' WHERE ISNULL(" + dt.Columns[0] + ", '') = '" + dr[0] + "'");
//extra lines and GO batch separator added between UPDATE statements for formating
sb.AppendLine(Environment.NewLine);
sb.AppendLine("GO");
sb.AppendLine(Environment.NewLine);
}
//output UPDATE commands as .sql script file
File.WriteAllText(sqlScript, sb.ToString());
}
Just to post an update of the code that I updated and that at the moment works perfectly. Thank you all for the answers and for helping me.
using System;
using System.IO;
using System.Text;
using System.Windows.Forms;
namespace GenererScriptSQL
{
public partial class Form1 : Form
{
public Form1()
{
InitializeComponent();
}
private void Form1_Load(object sender, EventArgs e)
{
}
private static void AddSqlCommand(StringBuilder sql, string[] columns, string[] types, string[] values)
{
sql.AppendLine("UPDATE Contact");
sql.AppendLine("SET");
//skip LoginId columns
for (int i = 1; i < columns.Length; i++)
{
switch (types[i].Trim())
{
case "int":
sql.Append($" Contact.{columns[i].Trim()} = {values[i]}");
break;
default:
sql.Append($" Contact.{columns[i].Trim()} = '{values[i]}'");
break;
}
if (columns.Length > 1 && i != columns.Length - 1)
{
sql.Append(",");
}
sql.AppendLine();
}
sql.AppendLine();
sql.AppendLine("WHERE");
sql.AppendLine();
sql.AppendLine($" Contact.{columns[0].Trim()} = '{values[0]}'");
sql.AppendLine();
}
private static StringBuilder GenerateSqlScript(string[] fileContent)
{
var sqlCommand = new StringBuilder();
string[] types = fileContent[0].Split(';');
string[] columns = fileContent[0].Split(';');
//skip the first line(header)
for (int i = 1; i < fileContent.Length; i++)
{
string[] values = fileContent[i].Split(';');
if (values.Length >= 1)
{
AddSqlCommand(sqlCommand, columns, types, values);
}
}
return sqlCommand;
}
private void buttonCreateSqlFile_Click(object sender, EventArgs e)
{
try
{
if (IsFileSelected())
{
string[] fileContent = File.ReadAllLines(textBoxFile.Text);
if (fileContent != null)
{
StringBuilder sqlCommand = GenerateSqlScript(fileContent);
if (!string.IsNullOrWhiteSpace(sqlCommand.ToString()))
{
WriteSqlFile(sqlCommand);
}
}
}
else
{
MessageBox.Show("Sélectionner le fichier de chargement.");
}
}
catch (Exception ex)
{
MessageBox.Show(ex.ToString());
}
}
private void buttonSelectFile_Click(object sender, EventArgs e)
{
try
{
using (var fileBrowser = new OpenFileDialog())
{
if (fileBrowser.ShowDialog() == DialogResult.OK)
{
textBoxFile.Text = fileBrowser.FileName;
}
}
}
catch (Exception ex)
{
MessageBox.Show(ex.ToString());
}
}
private bool IsFileSelected()
{
return !string.IsNullOrWhiteSpace(textBoxFile.Text) && File.Exists(textBoxFile.Text);
}
private void WriteSqlFile(StringBuilder sqlCommand)
{
var fileInfo = new FileInfo(textBoxFile.Text);
string BackupDate = fileInfo.Name + "_" + DateTime.Now.ToString("yyyy-MM-dd_HH-mm") + "_Update" + ".sql";
string sqlFilePath = Path.Combine(fileInfo.Directory.FullName, BackupDate);
if (File.Exists(sqlFilePath))
{
File.Delete(sqlFilePath);
}
File.WriteAllText(sqlFilePath, sqlCommand.ToString());
MessageBox.Show($#" Le fichier sql a été générée! {sqlFilePath}");
}
}
}
I am writing on my pdf-word converter and I just received a really strange exception witch makes no sens to me.
Error:PdfiumViewer.PdfException:{"Unsupported security scheme"}
Its the first time that such a exception appears. but I have to be honest that I never tried to convert more then 3-4 files from pdf to word and right now I am doing more then 100 files.
Here is my code I am sry if its too long but I simply do not know on which line the error occurs
public static void PdfToImage()
{
try
{
Application application = null;
application = new Application();
string path = #"C:\Users\chnikos\Desktop\Test\Batch1\";
foreach (string file in Directory.EnumerateFiles(path, "*.pdf"))
{
var doc = application.Documents.Add();
using (var document = PdfiumViewer.PdfDocument.Load(file))
{
int pagecount = document.PageCount;
for (int index = 0; index < pagecount; index++)
{
var image = document.Render(index, 200, 200, true);
image.Save(#"C:\Users\chnikos\Desktop\Test\Batch1\output" + index.ToString("000") + ".png", ImageFormat.Png);
application.Selection.InlineShapes.AddPicture(#"C:\Users\chnikos\Desktop\Test\Batch1\output" + index.ToString("000") + ".png");
}
string getFileName = file.Substring(file.LastIndexOf("\\"));
string getFileWithoutExtras = Regex.Replace(getFileName, #"\\", "");
string getFileWihtoutExtension = Regex.Replace(getFileWithoutExtras, #".pdf", "");
string fileName = #"C:\Users\chnikos\Desktop\Test\Batch1\" + getFileWihtoutExtension;
doc.PageSetup.PaperSize = WdPaperSize.wdPaperA4;
foreach (Microsoft.Office.Interop.Word.InlineShape inline in doc.InlineShapes)
{
.....
}
doc.PageSetup.TopMargin = 28.29f;
doc.PageSetup.LeftMargin = 28.29f;
doc.PageSetup.RightMargin = 30.29f;
doc.PageSetup.BottomMargin = 28.29f;
application.ActiveDocument.SaveAs(fileName, WdSaveFormat.wdFormatDocument);
doc.Close();
string imagePath = #"C:\Users\chnikos\Desktop\Test\Batch1\";
Array.ForEach(Directory.GetFiles(imagePath, "*.png"), delegate(string deletePath) { File.Delete(deletePath); });
}
}
}
catch (Exception e)
{
Console.WriteLine("Error: " + e);
}
}
}
}
I am using the below function to import a csv file into a datatable object csvData, however i face an oject reference issue that i'd like to understand more while using Regex.Replace to remove quote marks from the data:
private static DataTable Gettabledata(string cpath)
{
DataTable csvData = new DataTable();
try
{
using (TextFieldParser csvReader = new TextFieldParser(cpath))
{
csvReader.SetDelimiters(new string[] { "," });
csvReader.HasFieldsEnclosedInQuotes = true;
string[] colFields = csvReader.ReadFields();
foreach (string column in colFields)
{
DataColumn datecolumn = new DataColumn(column);
datecolumn.AllowDBNull = true;
csvData.Columns.Add(datecolumn);
}
while (!csvReader.EndOfData)
{
string[] fieldData = csvReader.ReadFields();
string pattern="\""; % remove quotation mark "
string replacement=""; % replace by empty.(eg "a", a)
Regex rgx = new Regex(pattern);
for (int i = 0; i < fieldData.Length; i++)
fieldData[i] = Regex.Replace(fieldData[i],pattern, replacement); %object reference issue
{
if (fieldData[i] == "")
{
fieldData[i] = null;
}
}
csvData.Rows.Add(fieldData);
}
}
}
catch (Exception ex)
{
}
return csvData;
}
The fieldData[i] = Regex.Replace(fieldData[i],pattern, replacement); is outside the {...} block. It should be inside it.
for (int i = 0; i < fieldData.Length; i++)
{
if (fieldData[i] == "")
{
fieldData[i] = null;
}
else
{
fieldData[i] = Regex.Replace(fieldData[i],pattern, replacement);
}
}