We Keep Coding

Reading csv without specific type

I want to read csv file and save to list or array or anything, but CsvHelper demands to save it as collection of specific type. Problem is my csv has so many columns, that mapping it to custom class will take a few weeks.
How can I just read it without saving as specific type? Accessing specific values with thing like row[1][2] is more than enough for me.

Add it to a datatable
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using System.Data;
namespace ConsoleApplication23
{
class Program
{
const string FILENAME = #"c:\temp\test.csv";
static void Main(string[] args)
{
StreamReader reader = new StreamReader(FILENAME);
string line = "";
DataTable dt = new DataTable();
int rowCount = 0;
while((line = reader.ReadLine()) != null)
{
line = line.Trim();
if (line.Length > 0)
{
string[] splitArray = line.Split(new char[] { ',' });
if (rowCount == 0)
{
foreach (string col in splitArray)
{
dt.Columns.Add(col, typeof(string));
}
}
else
{
dt.Rows.Add(splitArray);
}
}
}
}
}
}

When Exporting from sql to Csv -> How I change date format

I made my small project(c# app)your text, which export data from sql database into the csv file.
I would like to change date format from | 10.01.2022 to -> 2022.01.10.
Thanks for help.
Regards,
vit
using System.Data;
using System.Data.SqlClient;
namespace ExtractApp
{
class Program
{
static void Main(string[] args)
{
GetCSV();
}
private static string GetCSV()
{
using (SqlConnection cn = new SqlConnection(GetConnectionString()))
{
cn.Open();
return CreateCSV(new SqlCommand("Select * from [reports].[xxx]", cn).ExecuteReader());
cn.Close();
}
}
private static string CreateCSV(IDataReader reader)
{
string file = "C:\\SqlToCsv\\Data.csv";
List<string> lines = new List<string>();
string headerLine = "";
if (reader.Read())
{
string[] columns = new string[reader.FieldCount];
for (int i = 0; i < reader.FieldCount; i++)
{
columns[i] = reader.GetName(i);
}
headerLine = string.Join(",", columns);
lines.Add(headerLine);
}
//data
while (reader.Read())
{
object[] values = new object[reader.FieldCount];
reader.GetValues(values);
lines.Add(string.Join(",", values));
}
//file
System.IO.File.WriteAllLines(file, lines);
return file;
}
}
}

Since your column type is Date, it will be mapped to a DateTime (docs) in C#, so we can use pattern matching to get the DateTime and convert it to a string with the .ToString method:
object[] values = new object[reader.FieldCount];
reader.GetValues(values);
for (int i = 0; i < values.Length; ++i)
{
if (values[i] is DateTime dateTimeValue)
{
values[i] = dateTimeValue.ToString("yyyy.MM.dd", System.Globalization.CultureInfo.InvariantCulture);
}
}
lines.Add(string.Join(",", values));
More info:
Pattern matching
Standard date and time format strings
Custom date and time format strings

you can write it in sql syntax;
like if the name of date field is: created_date
In sql you can change it's format by:
`Select DATE_FORMAT(created_date, "%M.%m.%d"), other_fields then your condition `
Have a great time...

System.indexoutofrangeexception: 'cannot find column 1'

I have a program to parse a CSV file from local filesystem to a specified SQL Server table.
Now when i execute the program i get error :
System.IndexOutOfRangeException: 'Cannot find column 1' exception on the line where i the program attempts to populate the datatable.
On closer inspection the error shows that its emanating from row number 3 as shown on this link :
CSV_ERROR
This is how i am reading and saving the CSV file :
static void Main(string[] args)
{
var absPath = #"C:\Users\user\Documents\Projects\MastercardSurveillance\fbc_mc_all_cards.csv";
ProcessFile();
void ProcessFile()
{
string realPath = #"C:\Users\user\Documents\CSV";
string appLog = "CSVERRORS";
var logPath = realPath + Convert.ToString(appLog) + DateTime.Today.ToString("dd -MM-yy") + ".txt";
if (!File.Exists(logPath))
{
File.Create(logPath).Dispose();
}
var dt = GetDATATable();
if (dt == null)
{
return;
}
if (dt.Rows.Count == 0)
{
using (StreamWriter sw = File.AppendText(logPath))
{
sw.WriteLine("No rows imported after reading file " + absPath);
sw.Flush();
sw.Close();
}
return;
}
ClearData();
InsertDATA();
}
DataTable GetDATATable()
{
var FilePath = absPath;
string TableName = "Cards";
string realPath = #"C:\Users\user\Documents\CSV";
string appLog = "CSVERRORS";
var logPath = realPath + Convert.ToString(appLog) + DateTime.Today.ToString("dd -MM-yy") + ".txt";
if (!File.Exists(logPath))
{
File.Create(logPath).Dispose();
}
var dt = new DataTable(TableName);
using (var csvReader = new TextFieldParser(FilePath))
{
csvReader.SetDelimiters(new string[] { "," });
csvReader.HasFieldsEnclosedInQuotes = true;
var readFields = csvReader.ReadFields();
if (readFields == null)
{
using (StreamWriter sw = File.AppendText(logPath))
{
sw.WriteLine("Could not read header fields for file " + FilePath);
sw.Flush();
sw.Close();
}
return null;
}
foreach (var dataColumn in readFields.Select(column => new DataColumn(column, typeof(string)) { AllowDBNull = true, DefaultValue = string.Empty }))
{
dt.Columns.Add(dataColumn);
}
while (!csvReader.EndOfData)
{
var data = csvReader.ReadFields();
if (data == null)
{
using (StreamWriter sw = File.AppendText(logPath))
{
sw.WriteLine(string.Format("Could not read fields on line {0} for file {1}", csvReader.LineNumber, FilePath));
sw.Flush();
sw.Close();
}
continue;
}
var dr = dt.NewRow();
for (var i = 0; i < data.Length; i++)
{
if (!string.IsNullOrEmpty(data[i]))
{
dr[i] = data[i];
}
}
dt.Rows.Add(dr);
}
}
return dt;
}
void ClearData()
{
string SqlSvrConn = #"Server=XXXXXX-5QFK4BL\MSDEVOPS;Database=McardSurveillance;Trusted_Connection=True;MultipleActiveResultSets=true;";
using (var sqlConnection = new SqlConnection(SqlSvrConn))
{
sqlConnection.Open();
// Truncate the live table
using (var sqlCommand = new SqlCommand(_truncateLiveTableCommandText, sqlConnection))
{
sqlCommand.ExecuteNonQuery();
}
}
}
void InsertDATA()
{
string SqlSvrConn = #"Server=XXXXXX-5QFK4BL\MSDEVOPS;Database=McardSurveillance;Trusted_Connection=True;MultipleActiveResultSets=true;";
DataTable table = GetDATATable();
using (var sqlBulkCopy = new SqlBulkCopy(SqlSvrConn))
{
sqlBulkCopy.DestinationTableName = "dbo.Cards";
for (var count = 0; count < table.Columns.Count; count++)
{
sqlBulkCopy.ColumnMappings.Add(count, count);
}
sqlBulkCopy.WriteToServer(table);
}
}
}
How can i identify and possibly exclude the extra data columns being returned from the CSV file?
It appears there is a mismatch between number of columns in datatable and number of columns being read from the CSV file.
Im not sure however how i can account for this with my logic. For now i did not want to switch to using a CSV parse package but rather i need insight on how i can remove the extra column or rather ensure that the splitting takes account of all possible dubious characters.
For clarity i have a copy of the CSV file here :
CSV_FILE

How to iterate through a column in a csv file using CSVhelper library? using: dynamic object

CSV file
CSV file in notepad editor
using CsvHelper;
public class csvread
{
public dynamic APSSValue ;
public async Task GetMode()
{
try
{
FileOpenPicker openpicker = new FileOpenPicker();
openpicker.FileTypeFilter.Add(".csv");
IRandomAccessStreamWithContentType stream = await file.OpenReadAsync();
StreamReader reader = new StreamReader(stream.AsStream());
string UserCultureInfo = Thread.CurrentThread.CurrentCulture.Name;
CsvReader csv = new CsvReader(reader, culture: CultureInfo.CreateSpecificCulture(UserCultureInfo));
csv.Configuration.HasHeaderRecord = false;
csv.Configuration.Delimiter = ";";
while (csv.Read())
{
APSSValue = Enumerable.ToList(csv.GetRecord<dynamic>());
}
}
}
}
I tried this way shown above but with this I only get the number of columns.
I also tried
csv.Read();
APSSValue = Enumerable.ToList(csv.GetRecords<dynamic>());
but this gives me the entire data of csv file.
Questions:
I want to look for the value under the column (I_APSS_Modus) Please see the images shared above, It can be 0 or 1 so how can I look for that value if it is 0 or 1 ?
NOTE:
I don't want the values of entire column because all the values in the column would be either 0 or 1. So i just want one value from any row under that column.
CSV file is not same every time, so the column number for (I_APSS_Modus) will vary if the CSV file is different

Try following code which put data into a DataTable so you can easily filter with linq. Below will work with posted csv
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using System.Globalization;
using System.Data;
namespace ConsoleApplication8
{
class Program
{
const string FILENAME = #"c:\temp\test.txt";
static void Main(string[] args)
{
DataTable dt = new DataTable();
dt.Columns.Add("Timestamp", typeof(DateTime));
dt.Columns.Add("B_Kurze_Seite", typeof(Boolean));
dt.Columns.Add("I_Partikeleinfall Reinluft", typeof(int));
dt.Columns.Add("I_Partikeleinfall Rohluft", typeof(int));
dt.Columns.Add("I_APSS_Modus", typeof(int));
StreamReader reader = new StreamReader(FILENAME);
string line = "";
int row = 0;
string format = "yyyy MM dd HH:mm:ss:fff";
while ((line = reader.ReadLine()) != null)
{
line = line.Trim();
if (line.Length > 0 && !line.StartsWith("sep"))
{
if (++row > 1)
{
string[] splitRow = line.Split(new char[] { ';' });
dt.Rows.Add(new object[] {
DateTime.ParseExact(splitRow[0],format,CultureInfo.InvariantCulture),
(splitRow[1] == "FALSE")? false : true,
int.Parse(splitRow[2]),
int.Parse(splitRow[3]),
int.Parse(splitRow[4])
});
}
}
}
List<int> data = dt.AsEnumerable().Select(x => x.Field<int>("I_APSS_Modus")).ToList();
}
}
}
Here is code that will work with generic columns
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using System.Globalization;
using System.Data;
namespace ConsoleApplication8
{
class Program
{
const string FILENAME = #"c:\temp\test.txt";
static void Main(string[] args)
{
DataTable dt = new DataTable();
StreamReader reader = new StreamReader(FILENAME);
string line = "";
int row = 0;
string format = "yyyy MM dd HH:mm:ss:fff";
string[] columnNames = null;
while ((line = reader.ReadLine()) != null)
{
line = line.Trim();
if ((line.Length > 0) && !line.StartsWith("sep"))
{
string[] splitRow = line.Split(new char[] { ';' });
if (++row == 1)
{
columnNames = splitRow;
foreach (string col in splitRow)
{
switch(col)
{
case "Timestamp":
dt.Columns.Add(col, typeof(DateTime));
break;
case "I_APSS_Modus":
dt.Columns.Add(col, typeof(int));
break;
default:
dt.Columns.Add(col, typeof(string));
break;
}
}
}
else
{
DataRow newRow = dt.Rows.Add();
for (int i = 0; i < columnNames.Length; i++)
{
switch (columnNames[i])
{
case "Timestamp":
newRow["Timestamp"] = DateTime.ParseExact(splitRow[i], format, CultureInfo.InvariantCulture);
break;
case "I_APSS_Modus":
newRow["I_APSS_Modus"] = int.Parse(splitRow[i]);
break;
default:
newRow[i] = splitRow[i];
break;
}
}
}
}
}
List<int> data = dt.AsEnumerable().Select(x => x.Field<int>("I_APSS_Modus")).ToList();
}
}
}

If you want to stay with CsvHelper, a simple mapping should do the trick:
// Data model of a row (only I_APSS_Modul is of interest)
public class DataSet
{
public int ApssModus { get; set; }
}
// Mapping
public class ApssMap : ClassMap<DataSet>
{
public ApssMap()
{
Map(m => m.ApssModus).Name("I_APSS_Modus");
}
}
// Reading records
public void GetMode()
{
string UserCultureInfo = Thread.CurrentThread.CurrentCulture.Name;
using (var tr = new StreamReader(new FileStream("any.csv", FileMode.Open)))
{
using (var csv = new CsvReader(tr, new CsvConfiguration(CultureInfo.CreateSpecificCulture(UserCultureInfo))
{
Delimiter = ";"
}))
{
csv.Configuration.RegisterClassMap<ApssMap>();
var records = csv.GetRecords<DataSet>().ToList();
}
}
}
Edit: Check the official docs for all the mapping options: https://joshclose.github.io/CsvHelper/examples/configuration/class-maps

If you just want to read the first row of data, you have to read the header first and then you can read the first data row.
static void Main(string[] args)
{
using (var stream = new MemoryStream())
using (var writer = new StreamWriter(stream))
using (var reader = new StreamReader(stream))
using (var csv = new CsvReader(reader, CultureInfo.InvariantCulture))
{
writer.WriteLine("Timestamp;B_Kurze_Seite;I_Parikeleinfall Reinluft;I_Partikeleinfall Rohluft;I_APSS_Modus");
writer.WriteLine("2020 06 27 08:49:20:335;FALSE;15;0;0");
writer.WriteLine("2020 06 27 08:49:20:391;FALSE;0;0;0");
writer.Flush();
stream.Position = 0;
csv.Configuration.Delimiter = ";";
csv.Read();
csv.ReadHeader();
csv.Read();
var record = csv.GetRecord<dynamic>();
var APSSModus = record.I_APSS_Modus;
}
}
Edit: This should give you the same answer as #jdweng
static void Main(string[] args)
{
using (var stream = new MemoryStream())
using (var writer = new StreamWriter(stream))
using (var reader = new StreamReader(stream))
using (var csv = new CsvReader(reader, CultureInfo.InvariantCulture))
{
writer.WriteLine("sep=;");
writer.WriteLine("");
writer.WriteLine("");
writer.WriteLine("");
writer.WriteLine("Timestamp;B_Kurze_Seite;I_Parikeleinfall Reinluft;I_Partikeleinfall Rohluft;I_APSS_Modus");
writer.WriteLine("2020 06 27 08:49:20:335;FALSE;15;0;0");
writer.WriteLine("2020 06 27 08:49:20:391;FALSE;0;0;0");
writer.Flush();
stream.Position = 0;
csv.Configuration.Delimiter = ";";
csv.Configuration.ShouldSkipRecord = row => row[0].StartsWith("sep=") || row.All(string.IsNullOrEmpty);
var records = csv.GetRecords<dynamic>();
List<int> data = records.Select(x => (int)int.Parse(x.I_APSS_Modus)).ToList();
}
}

How to create a generic text file parser for any find of text file?

Want to create a generic text file parser in c# for any find of text file.Actually i have 4 application all 4 getting input data from txt file format but text files are not homogeneous in nature.i have tried fixedwithdelemition.
private static DataTable FixedWidthDiliminatedTxtRead()
{
string[] fields;
StringBuilder sb = new StringBuilder();
List<StringBuilder> lst = new List<StringBuilder>();
DataTable dtable = new DataTable();
ArrayList aList;
using (TextFieldParser tfp = new TextFieldParser(testOCC))
{
tfp.TextFieldType = FieldType.FixedWidth;
tfp.SetFieldWidths(new int[12] { 2,25,8,12,13,5,6,3,10,11,10,24 });
for (int col = 1; col < 13; ++col)
dtable.Columns.Add("COL" + col);
while (!tfp.EndOfData)
{
fields = tfp.ReadFields();
aList = new ArrayList();
for (int i = 0; i < fields.Length; ++i)
aList.Add(fields[i] as string);
if (dtable.Columns.Count == aList.Count)
dtable.Rows.Add(aList.ToArray());
}
}
return dtable;
}
but i feel its very rigid one and really varies application to application making it configgurable .any better way ..
tfp.SetFieldWidths(new int[12] { 2,25,8,12,13,5,6,3,10,11,10,24 });
File nature :
Its a report kind of file .
position of columns are very similar
row data of file id different .
I get this as a reference
http://www.codeproject.com/Articles/11698/A-Portable-and-Efficient-Generic-Parser-for-Flat-F
any other thoughts ?

If the only thing different is the field widths, you could just try sending the field widths in as a parameter:
private static DataTable FixedWidthDiliminatedTxtRead(int[] fieldWidthArray)
{
string[] fields;
StringBuilder sb = new StringBuilder();
List<StringBuilder> lst = new List<StringBuilder>();
DataTable dtable = new DataTable();
ArrayList aList;
using (TextFieldParser tfp = new TextFieldParser(testOCC))
{
tfp.TextFieldType = FieldType.FixedWidth;
tfp.SetFieldWidths(fieldWidthArray);
for (int col = 1; col < 13; ++col)
dtable.Columns.Add("COL" + col);
while (!tfp.EndOfData)
{
fields = tfp.ReadFields();
aList = new ArrayList();
for (int i = 0; i < fields.Length; ++i)
aList.Add(fields[i] as string);
if (dtable.Columns.Count == aList.Count)
dtable.Rows.Add(aList.ToArray());
}
}
return dtable;
}
If you will have more logic to grab the data, you might want to consider defining an interface or abstract class for a GenericTextParser and create concrete implementations for each other file.

Hey I made one of these last week.
I did not write it with the intentions of other people using it so I appologize in advance if its not documented well but I cleaned it up for you. ALSO I grabbed several segments of code from stack overflow so I am not the original author of several pieces of this.
The places you need to edit are the path and pathout and the seperators of text.
char[] delimiters = new char[]
So it searches for part of a word and then grabs the whole word. I used a c# console application for this.
Here you go:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.IO;
namespace UniqueListofStringFinder
{
class Program
{
static void Main(string[] args)
{
string path = #"c:\Your Path\in.txt";
string pathOut = #"c:\Your Path\out.txt";
string data = "!";
Console.WriteLine("Current Path In is set to: " + path);
Console.WriteLine("Current Path Out is set to: " + pathOut);
Console.WriteLine(Environment.NewLine + Environment.NewLine + "Input String to Search For:");
Console.Read();
string input = Console.ReadLine();
// Delete the file if it exists.
if (!File.Exists(path))
{
// Create the file.
using (FileStream fs = File.Create(path))
{
Byte[] info =
new UTF8Encoding(true).GetBytes("This is some text in the file.");
// Add some information to the file.
fs.Write(info, 0, info.Length);
}
}
System.IO.StreamReader file = new System.IO.StreamReader(path);
List<string> Spec = new List<string>();
using (StreamReader sr = File.OpenText(path))
{
while (!file.EndOfStream)
{
string s = file.ReadLine();
if (s.Contains(input))
{
char[] delimiters = new char[] { '\r', '\n', '\t', ')', '(', ',', '=', '"', '\'', '<', '>', '$', ' ', '#', '[', ']' };
string[] parts = s.Split(delimiters,
StringSplitOptions.RemoveEmptyEntries);
foreach (string word in parts)
{
if (word.Contains(input))
{
if( word.IndexOf(input) == 0)
{
Spec.Add(word);
}
}
}
}
}
Spec.Sort();
// Open the stream and read it back.
//while ((s = sr.ReadLine()) != null)
//{
// Console.WriteLine(s);
//}
}
Console.WriteLine();
StringBuilder builder = new StringBuilder();
foreach (string s in Spec) // Loop through all strings
{
builder.Append(s).Append(Environment.NewLine); // Append string to StringBuilder
}
string result = builder.ToString(); // Get string from StringBuilder
Program a = new Program();
data = a.uniqueness(result);
int i = a.writeFile(data,pathOut);
}
public string uniqueness(string rawData )
{
if (rawData == "")
{
return "Empty Data Set";
}
List<string> dataVar = new List<string>();
List<string> holdData = new List<string>();
bool testBool = false;
using (StringReader reader = new StringReader(rawData))
{
string line;
while ((line = reader.ReadLine()) != null)
{
foreach (string s in holdData)
{
if (line == s)
{
testBool = true;
}
}
if (testBool == false)
{
holdData.Add(line);
}
testBool = false;
// Do something with the line
}
}
int i = 0;
string dataOut = "";
foreach (string s in holdData)
{
dataOut += s + "\r\n";
i++;
}
// Write the string to a file.
return dataOut;
}
public int writeFile(string dataOut, string pathOut)
{
try
{
System.IO.StreamWriter file = new System.IO.StreamWriter(pathOut);
file.WriteLine(dataOut);
file.Close();
}
catch (Exception ex)
{
dataOut += ex.ToString();
return 1;
}
return 0;
}
}
}

private static DataTable FixedWidthTxtRead(string filename, int[] fieldWidths)
{
string[] fields;
DataTable dtable = new DataTable();
ArrayList aList;
using (TextFieldParser tfp = new TextFieldParser(filename))
{
tfp.TextFieldType = FieldType.FixedWidth;
tfp.SetFieldWidths(fieldWidths);
for (int col = 1; col <= fieldWidths.length; ++col)
dtable.Columns.Add("COL" + col);
while (!tfp.EndOfData)
{
fields = tfp.ReadFields();
aList = new ArrayList();
for (int i = 0; i < fields.Length; ++i)
aList.Add(fields[i] as string);
if (dtable.Columns.Count == aList.Count) dtable.Rows.Add(aList.ToArray());
}
}
return dtable;
}

Here's what I did:
I built a factory for the type of processor needed (based on file type/format), which abstracted the file reader.
I then built a collection object that contained a set of triggers for each field I was interested in (also contained the property name for which this field is destined). This settings collection is loaded in via an XML configuration file, so all I need to change are the settings, and the base parsing process can react to how the settings are configured. Finally I built a reflection wrapper wherein once a field is parsed, the corresponding property on the model object is set.
As the file flowed through, the triggers for each setting evaluated each lines value. When it found what it was set to find (via pattern matching, or column length values) it fired and event that bubbled up and set a property on the model object. I can show some pseudo code if you're interested. It needs some work for efficiency's sake, but I like the concept.