Please see the below code which I use for writing to a text file. But database fetches nearly 1 millions records, so can someone please advise a faster way to do this or how should I change the below mentioned code to get it working faster?
try
{
using (OleDbConnection connection = new OleDbConnection(ConnectionString))
{
OleDbCommand command = new OleDbCommand(queryString, connection);
connection.Open();
OleDbDataReader reader = command.ExecuteReader();
DataTable dt = new DataTable();
dt.Load(reader);
StreamWriter writer = new StreamWriter(FilePath + FileName);
var result = string.Empty;
for (int i = 0; i < dt.Rows.Count; i++)
{
for (int j = 0; j < dt.Columns.Count; j++)
{
result += dt.Rows[i][j] + "|";
}
result += "\r\n";
}
writer.WriteLine(result);
reader.Close();
writer.Close();
Dts.TaskResult = (int)ScriptResults.Success;
}
}
You load 1 million records in a datatable and then loop again on that table to write each row.
Probably you could do the same thing, but in half the time if you read a single row and write it down:
OleDbDataReader reader = command.ExecuteReader();
StreamWriter writer = new StreamWriter(FilePath + FileName);
var result = string.Empty;
while(reader.Read())
{
for (int j = 0; j < reader.FieldCount; j++)
{
result += reader[j] + "|";
}
result += "\r\n";
}
writer.WriteLine(result);
Also, using a StringBuilder to buffer your read in memory and not constantly writing to disk could be very beneficial in this situation:
// This size is just for example purpose. Should be fine tuned
StringBuilder buffer = new StringBuilder(1048576);
while(reader.Read())
{
for (int j = 0; j < reader.FieldCount; j++)
{
buffer.Append(reader[j] + "|");
}
buffer.AppendLine();
if(buffer.Length > 1048576 - 1024)
{
writer.Write(buffer.ToString());
buffer.Length = 0;
}
}
writer.Write(buffer.ToString());
Related
I have a stored procedure that has multiple select statements using temp tables. I want to copy the results into one CSV file. Each result has different columns. I would like to copy them in such a way that each result set should leave two lines of space in CSV file.
Example below :
Sample stored procedure
Create procedure usp_Test_CSV_Report
As
Begin
select 'Text Sample' as Description, 123 Amount, 20210511 as Joindate
select GETDATE() as MonthATB
select 1 as AccountId, 'CI' as Name
select 'Sample Report'
End
The temp tables have been created within the stored procedure which will be called like Select * from #temp. I have not included real stored procedure which is vast.
I will be running the stored procedure using C#
string query = "EXEC alpha.dbo.usp_Test_CSV_Report";
SqlCommand cmd = new SqlCommand(query, SQLConnection);
SQLConnection.Open();
DataTable d_table = new DataTable();
SqlDataReader sqlReader = cmd.ExecuteReader();
while (sqlReader.Read())
{
d_table.Load(sqlReader);
// Write the Header Row to File
int ColumnCount = d_table.Columns.Count;
for (int ic = 0; ic < ColumnCount; ic++)
{
//MessageBox.Show(d_table.Columns[ic].ToString());
sw.Write(d_table.Columns[ic]);
if (ic < ColumnCount - 1)
{
sw.Write(FileDelimiter);
}
}
sw.Write(sw.NewLine);
// Write All Rows to the File
foreach (DataRow dr in d_table.Rows)
{
for (int ir = 0; ir < ColumnCount; ir++)
{
if (!Convert.IsDBNull(dr[ir]))
{
sw.Write(dr[ir].ToString());
//MessageBox.Show(dr[ir].ToString());
}
if (ir < ColumnCount - 1)
{
sw.Write(FileDelimiter);
}
}
sw.Write(sw.NewLine);
}
}
sqlReader.NextResult();
while (sqlReader.Read())
{
d_table.Load(sqlReader);
// Write the Header Row to File
int ColumnCount = d_table.Columns.Count;
for (int ic = 0; ic < ColumnCount; ic++)
{
//MessageBox.Show(d_table.Columns[ic].ToString());
sw.Write(d_table.Columns[ic]);
if (ic < ColumnCount - 1)
{
sw.Write(FileDelimiter);
}
}
sw.Write(sw.NewLine);
// Write All Rows to the File
foreach (DataRow dr in d_table.Rows)
{
for (int ir = 0; ir < ColumnCount; ir++)
{
if (!Convert.IsDBNull(dr[ir]))
{
sw.Write(dr[ir].ToString());
//MessageBox.Show(dr[ir].ToString());
}
if (ir < ColumnCount - 1)
{
sw.Write(FileDelimiter);
}
}
sw.Write(sw.NewLine);
}
}
SQLConnection.Close();
sw.Close();
So far I have tried this but this is not working!!
Any help?
I maintain a nuget package, Sylvan.Data.Csv, that makes this very easy.
string query = "EXEC alpha.dbo.usp_Test_CSV_Report";
using SqlConnection conn = GetSqlConnection();
conn.Open();
using SqlCommand cmd = new SqlCommand(query, conn);
using var sw = File.CreateText("usp_Test_CSV_Report.csv");
using var csvWriter = CsvDataWriter.Create(sw);
using var sqlReader = cmd.ExecuteReader();
bool first = true;
do
{
if (!first)
{
// write the two lines to separate the result sets.
sw.WriteLine();
sw.WriteLine();
}
first = false;
csvWriter.Write(sqlReader);
} while (sqlReader.NextResult());
The library also supports reading multiple result sets out of a single CSV in much the same way:
// tell the reader to expect multiple result sets.
var csvOpts = new CsvDataReaderOptions { ResultSetMode = ResultSetMode.MultiResult };
var csvReader = CsvDataReader.Create("usp_Test_CSV_Report.csv", csvOpts);
do
{
while (csvReader.Read())
{
for(int i = 0; i < csvReader.FieldCount; i++)
{
var value = csvReader.GetString(i);
}
}
} while (csvReader.NextResult());
I am trying to export data into a CSV file from the SQL server. The code from this link (Export SQL Server Data into CSV file) is working with some except. In some rows that contain commas, the table arrangement is not correct. The code i have try
using (var connection = ConnectionToSqlServer.GetConnection())
{
connection.Open();
SqlCommand sqlCmd = new SqlCommand("Select * from dbo.Test", connection);
SqlDataReader reader = sqlCmd.ExecuteReader();
string fileName = "test.csv";
StreamWriter sw = new StreamWriter(fileName);
object[] output = new object[reader.FieldCount];
for (int i = 0; i < reader.FieldCount; i++)
{
for (int i = 0; i < reader.FieldCount; i++)
{
if (reader.GetName(i).Contains(","))
{
output[i] = "\"" + reader.GetName(i) + "\"";
}
else
output[i] = reader.GetName(i);
}
}
sw.WriteLine(string.Join(",", output));
while (reader.Read())
{
reader.GetValues(output);
sw.WriteLine(string.Join(",", output));
}
sw.Close();
reader.Close();
connection.Close();
}
I am suggesting you consider below options:
Quote the values, to have proper CSV generation. If the CSV content has , inside it, then the generated CSV might be having an issue.
while (reader.Read())
{
reader.GetValues(output);
sw.WriteLine(string.Join(",", $"\"{output}\""));
}
You can think of using library like CSVHelper
I'm generating a csv file from an SqlDataReader, however it is not writing the column names, how can I make it write them? The code I'm using is as follows:
SqlConnection conn = new SqlConnection(myconn);
SqlCommand cmd = new SqlCommand("dbo.test", conn);
cmd.CommandType = CommandType.StoredProcedure;
conn.Open();
SqlDataReader reader = cmd.ExecuteReader();
StringBuilder sb = new StringBuilder();
StreamWriter sw = new StreamWriter(myfilePath + "testfile.csv");
while (reader.Read())
{
for (int i = 0; i < reader.FieldCount; i++)
{
string value = reader[i].ToString();
if (value.Contains(","))
value = "\"" + value + "\"";
sb.Append(value.Replace(Environment.NewLine, " ") + ",");
}
sb.Length--; // Remove the last comma
sb.AppendLine();
}
conn.Close();
sw.Write(sb.ToString());
sw.Close();
Read all the column names and append it to sb then iterate reader.
SqlDataReader reader = cmd.ExecuteReader();
StringBuilder sb = new StringBuilder();
//Get All column
var columnNames = Enumerable.Range(0, reader.FieldCount)
.Select(reader.GetName) //OR .Select("\""+ reader.GetName"\"")
.ToList();
//Create headers
sb.Append(string.Join(",", columnNames));
//Append Line
sb.AppendLine();
while (reader.Read())
....
Using this solution i created an extension.
/// <summary>
///
/// </summary>
/// <param name="reader"></param>
/// <param name="filename"></param>
/// <param name="path">if null/empty will use IO.Path.GetTempPath()</param>
/// <param name="extension">will use csv by default</param>
public static void ToCsv(this IDataReader reader, string filename, string path = null, string extension = "csv")
{
int nextResult = 0;
do
{
var filePath = Path.Combine(string.IsNullOrEmpty(path) ? Path.GetTempPath() : path, string.Format("{0}.{1}", filename, extension));
using (StreamWriter writer = new StreamWriter(filePath))
{
writer.WriteLine(string.Join(",", Enumerable.Range(0, reader.FieldCount).Select(reader.GetName).ToList()));
int count = 0;
while (reader.Read())
{
writer.WriteLine(string.Join(",", Enumerable.Range(0, reader.FieldCount).Select(reader.GetValue).ToList()));
if (++count % 100 == 0)
{
writer.Flush();
}
}
}
filename = string.Format("{0}-{1}", filename, ++nextResult);
}
while (reader.NextResult());
}
You can use SqlDataReader.GetName to get the column name
for (int i = 0; i < reader.FieldCount; i++)
{
string columnName = reader.GetName(i);
}
Also you can create an extension method like below:
public static List<string> ToCSV(this IDataReader dataReader, bool includeHeaderAsFirstRow, string separator)
{
List<string> csvRows = new List<string>();
StringBuilder sb = null;
if (includeHeaderAsFirstRow)
{
sb = new StringBuilder();
for (int index = 0; index < dataReader.FieldCount; index++)
{
if (dataReader.GetName(index) != null)
sb.Append(dataReader.GetName(index));
if (index < dataReader.FieldCount - 1)
sb.Append(separator);
}
csvRows.Add(sb.ToString());
}
while (dataReader.Read())
{
sb = new StringBuilder();
for (int index = 0; index < dataReader.FieldCount - 1; index++)
{
if (!dataReader.IsDBNull(index))
{
string value = dataReader.GetValue(index).ToString();
if (dataReader.GetFieldType(index) == typeof(String))
{
//If double quotes are used in value, ensure each are replaced but 2.
if (value.IndexOf("\"") >= 0)
value = value.Replace("\"", "\"\"");
//If separtor are is in value, ensure it is put in double quotes.
if (value.IndexOf(separator) >= 0)
value = "\"" + value + "\"";
}
sb.Append(value);
}
if (index < dataReader.FieldCount - 1)
sb.Append(separator);
}
if (!dataReader.IsDBNull(dataReader.FieldCount - 1))
sb.Append(dataReader.GetValue(dataReader.FieldCount - 1).ToString().Replace(separator, " "));
csvRows.Add(sb.ToString());
}
dataReader.Close();
sb = null;
return csvRows;
}
Example:
List<string> rows = null;
using (SqlDataReader dataReader = command.ExecuteReader())
{
rows = dataReader.ToCSV(includeHeadersAsFirstRow, separator);
dataReader.Close();
}
You can use the SqlDataReader.GetName method to get the name of a column, like this:
for(int i = 0; i < reader.FieldCount; i++)
{
string columnName = reader.GetName(i);
}
I developed following high performance extension
static void Main(string[] args)
{
SqlConnection sqlCon = new SqlConnection("Removed");
sqlCon.Open();
SqlCommand sqlCmd = new SqlCommand("Select * from Table", sqlCon);
SqlDataReader reader = sqlCmd.ExecuteReader();
string csv=reader.ToCSVHighPerformance(true);
File.WriteAllText("Test.CSV", csv);
reader.Close();
sqlCon.Close();
}
Extention:
public static string ToCSVHighPerformance(this IDataReader dataReader, bool includeHeaderAsFirstRow = true,
string separator = ",")
{
DataTable dataTable = new DataTable();
StringBuilder csvRows = new StringBuilder();
string row = "";
int columns ;
try
{
dataTable.Load(dataReader);
columns= dataTable.Columns.Count;
//Create Header
if (includeHeaderAsFirstRow)
{
for (int index = 0; index < columns; index++)
{
row += (dataTable.Columns[index]);
if (index < columns - 1)
row += (separator);
}
row += (Environment.NewLine);
}
csvRows.Append(row);
//Create Rows
for (int rowIndex = 0; rowIndex < dataTable.Rows.Count; rowIndex++)
{
row = "";
//Row
for (int index = 0; index < columns - 1; index++)
{
string value = dataTable.Rows[rowIndex][index].ToString();
//If type of field is string
if (dataTable.Rows[rowIndex][index] is string)
{
//If double quotes are used in value, ensure each are replaced by double quotes.
if (value.IndexOf("\"") >= 0)
value = value.Replace("\"", "\"\"");
//If separtor are is in value, ensure it is put in double quotes.
if (value.IndexOf(separator) >= 0)
value = "\"" + value + "\"";
//If string contain new line character
while (value.Contains("\r"))
{
value = value.Replace("\r", "");
}
while (value.Contains("\n"))
{
value = value.Replace("\n", "");
}
}
row += value;
if (index < columns - 1)
row += separator;
}
dataTable.Rows[rowIndex][columns - 1].ToString().ToString().Replace(separator, " ");
row += Environment.NewLine;
csvRows.Append(row);
}
}
catch (Exception ex)
{
throw ex;
}
return csvRows.ToString();
}
This question already has answers here:
Closed 10 years ago.
Possible Duplicate:
Efficient Method for Creating CSV String from Lists/SortedLists C#?
Here i need to export database information to the .csv file using c#,
Here i am giving my code, but its taking more time loading to the csv file
public void CSVFile(DataTable table, string FilePath)
{
// Create the CSV file to which grid data will be exported.
StreamWriter swr = new StreamWriter(FilePath, false);
//First we will write the headers.
int iColCount = table.Columns.Count;
for (int i = 0; i < iColCount; i++)
{
swr.Write(dtDataTablesList.Columns[i]);
if (i < iColCount - 1)
{
swr.Write(",");
}
}
swr.Write(swr.NewLine);
// Now write all the rows.
foreach (DataRow dr in table.Rows)
{
for (int i = 0; i < iColCount; i++)
{
if (!Convert.IsDBNull(dr[i]))
{
swr.Write(dr[i].ToString());
}
if (i < iColCount - 1)
{
swr.Write(",");
}
}
swr.Write(sw.NewLine);
}
swr.Close();
}
Please tell me better way of doing
Thanks
Yes recently i had the same issue,
i have the solution, try this code.
private void button1_Click(object sender, EventArgs e)
{
Stopwatch swra = new Stopwatch();
swra.Start();
string NewconnectionString = "myCoonectionString";
StreamWriter CsvfileWriter = new StreamWriter(#"D:\testfile.csv");
string sqlselectQuery = "select * from Mytable";
SqlCommand sqlcmd = new SqlCommand();
SqlConnection spContentConn = new SqlConnection(NewconnectionString);
sqlcmd.Connection = spContentConn;
sqlcmd.CommandTimeout = 0;
sqlcmd.CommandType = CommandType.Text;
sqlcmd.CommandText = sqlselectQuery;
spContentConn.Open();
using (spContentConn)
{
using (SqlDataReader sdr = sqlcmd.ExecuteReader())
using (CsvfileWriter)
{
//For getting the Table Headers
DataTable Tablecolumns = new DataTable();
for (int i = 0; i < sdr.FieldCount; i++)
{
Tablecolumns.Columns.Add(sdr.GetName(i));
}
CsvfileWriter.WriteLine(string.Join(",", Tablecolumns.Columns.Cast<datacolumn>().Select(csvfile => csvfile.ColumnName)));
//For table headers
while (sdr.Read())
//based on your columns
YourWriter.WriteLine(sdr[0].ToString() + "," + sdr[1].ToString() + "," +
sdr[2].ToString() + "," + sdr[3].ToString() + "," + sdr[4].ToString() + "," +
sdr[5].ToString() + "," + sdr[6].ToString() + "," + sdr[7].ToString() + "," +
sdr[8].ToString() + "," + sdr[9].ToString() + "," + sdr[10].ToString() + "," +
sdr[11].ToString() + ",");
}
}
swra.Stop();
Console.WriteLine(swra.ElapsedMilliseconds);
}
with this code we can export database information to the csv file speedly using datareader.
in line
urls[i] = Reader.GetValue(i).ToString();
it say Error 1 The name 'i' does not exist in the current context
how can I fix it
private void Form1_Load(object sender, EventArgs e)
{
string MyConString = "SERVER=192.168.0.78;" +
"DATABASE=webboard;" +
"UID=aimja;" +
"PASSWORD=aimjawork;" +
"charset=utf8;";
MySqlConnection connection = new MySqlConnection(MyConString);
MySqlCommand command = connection.CreateCommand();
MySqlDataReader Reader;
command.CommandText = "SELECT url FROM `listweb` WHERE `url` IS NOT NULL AND ( `webbordkind` = '¿¿¿¿¿¿¿¿¿¿¿¿' ) and `nourl`= 'n' order by province, amphore limit 4 ";
connection.Open();
Reader = command.ExecuteReader();
string[] urls = new string[2];
string thisrow = "";
string sumthisrow = "";
string urlname ;
while (Reader.Read())
{
thisrow = "";
for (int i = 0; i < Reader.FieldCount; i++)
thisrow += Reader.GetValue(i).ToString();
urlname = Reader.GetValue(i).ToString();
urls[i] = Reader.GetValue(i).ToString();
// System.IO.File.AppendAllText(#"C:\file.txt", thisrow + " " + Environment.NewLine);
sumthisrow = sumthisrow + thisrow;
You need to add braces to your for loop otherwise it only loops the first statement.
for (int i = 0; i < Reader.FieldCount; i++)
{
thisrow += Reader.GetValue(i).ToString();
urlname = Reader.GetValue(i).ToString();
urls[i] = Reader.GetValue(i).ToString();
}
You are missing braces here:
for (int i = 0; i < Reader.FieldCount; i++)
{
thisrow += Reader.GetValue(i).ToString();
urlname = Reader.GetValue(i).ToString();
urls[i] = Reader.GetValue(i).ToString();
}
I'd also advise you not to create strings by concatenating in a loop. Put them in a List<string> first then at the end convert it to an array (except in .NET 4.0 or newer where this step is not required) and use string.Join. As well as giving better performance this allows allows you to add a separator between the fields, which I assume you want...
If you don't need a separator then you can use a StringBuilder.
Your braces are missing for the FOR loop. The variable i is available only within the FOR loop which is only one line after your loop in your case.
for (int i = 0; i < Reader.FieldCount; i++)
{
thisrow += Reader.GetValue(i).ToString();
urlname = Reader.GetValue(i).ToString();
urls[i] = Reader.GetValue(i).ToString();
}