We Keep Coding

selecting all photographs from sql database

I want to export all photographs from our database into a data table. I will then loop through the table and save each image to disk. There are approx 7000 photos.
When I start the process I can retrieve around 4000 photographs before I start to get error messages like, Exception of type 'System.OutOfMemoryException' was thrown.
If I change the SQL query to retrieve half as many photographs, say 3500 then the process completes successfully.
While I have now achieved what I wanted by modifying the SQL each time I run the code, I would like to improve my code so that all 7000 photographs are returned. Could somebody please advise on a better process.
Here is my method
public static DataTable GetAllPhotos()
{
DataTable dt = new DataTable();
dt.Columns.Add("personId", typeof(string));
dt.Columns.Add("Photo", typeof(Bitmap));
string SQL = "";
byte[] getImg = new byte[0];
byte[] BitmapImg = new byte[0];
string personId = "";
SqlConnection conn = new SqlConnection();
conn.ConnectionString = _connString;
SQL = #"select per.person_id,pho.photo
from person as per
left join photo as pho on per.photo_id = pho.photo_id
where photo is not null";
conn.Open();
SqlDataReader dr = null;
SqlCommand cmd = new SqlCommand(SQL, conn);
dr = cmd.ExecuteReader();
while (dr.Read())
{
try
{
getImg = (byte[])dr["Photo"];
personId = Convert.ToString(dr["person_id"]);
MemoryStream str = new MemoryStream(getImg);
Bitmap bitmap = new Bitmap(Image.FromStream(str));
BitmapImg = ImageToByte(bitmap);
dt.Rows.Add(personId, bitmap);
}
catch (Exception ex)
{
LogWriter.WriteLine(personId + ex.Message.ToString());
}
}
conn.Close();
return dt;
}

If your intent is saving images to disk, then why would you get them into an intermediate datatable? Wouldn't it be better if you directly write out to disk as you read them? If so, assuming those are .bmp files:
public static void DumpAllPhotos()
{
string sql = #"select per.person_id,pho.photo
from person as per
inner join photo as pho on per.photo_id = pho.photo_id";
string folder = #"c:\MyFolder"; // output folder
using (SqlConnection con = new SqlConnection(_connString))
using (SqlCommand cmd = new SqlCommand(sql,con))
{
con.Open();
var rdr = cmd.ExecuteReader();
while (rdr.Read())
{
var bytes = (byte[])rdr["photo"];
var path = Path.Combine(folder, $"{rdr["person_id"].ToString()}.bmp");
File.WriteAllBytes(path, bytes);
}
con.Close();
}
}

Bitmap has to be disposed, you're using too many handles.
So your while loop should be something like this:
while (dr.Read())
{
try
{
getImg = (byte[])dr["Photograph"];
personId = Convert.ToString(dr["person_id"]);
MemoryStream str = new MemoryStream(getImg);
Bitmap bitmap = new Bitmap(Image.FromStream(str));
BitmapImg = ImageToByte(bitmap);
dt.Rows.Add(personId, bitmap);
bitmap.Dipose(); // <--- DISPOSE!!
}
catch (Exception ex)
{
LogWriter.WriteLine(personId + ex.Message.ToString());
}
}
or maybe even better:
while (dr.Read())
{
try
{
getImg = (byte[])dr["Photograph"];
personId = Convert.ToString(dr["person_id"]);
MemoryStream str = new MemoryStream(getImg);
using (Bitmap bitmap = new Bitmap(Image.FromStream(str))) {
BitmapImg = ImageToByte(bitmap);
dt.Rows.Add(personId, bitmap);
}
}
catch (Exception ex)
{
LogWriter.WriteLine(personId + ex.Message.ToString());
}
}

i am using DigitalPeronsa U ARE U4500. i am facing problem in saving fingerprint into mysql using c#

I am using MySql database and column type is VARBINARY. I am trying to save template into database. Stored successfully but in database it is storing "System.Byte[]" but I want to store the binary data. Can you tell why this is displaying in database? is there in code problem or in database problem?
protected override void Process(DPFP.Sample Sample)
{
base.Process(Sample);
DPFP.FeatureSet features = ExtractFeatures(Sample, DPFP.Processing.DataPurpose.Enrollment);
DPFP.Capture.SampleConversion ToByte = new DPFP.Capture.SampleConversion();
if (features != null) try
{
MakeReport("The fingerprint feature set was created.");
Enroller.AddFeatures(features);
}
finally
{
UpdateStatus();
// Check if template has been created.
switch (Enroller.TemplateStatus)
{
case DPFP.Processing.Enrollment.Status.Ready: // report success and stop capturing
{
OnTemplate(Enroller.Template);
SetPrompt("Click Close, and then click Fingerprint Verification.");
Stop();
MemoryStream fingerprintData = new MemoryStream();
Enroller.Template.Serialize(fingerprintData);
fingerprintData.Position = 0;
BinaryReader br = new BinaryReader(fingerprintData);
byte[] bytes = br.ReadBytes((Int32)fingerprintData.Length);
try
{
MySqlConnection conn = new MySqlConnection("server=localhost;username=root;password=root;database=enroll");
conn.Open();
MySqlCommand cmd = new MySqlCommand();
cmd.Connection = conn;
cmd.CommandText = "INSERT INTO reg(reg_temp) VALUES('" + bytes + "')";
cmd.ExecuteNonQuery();
conn.Close();
MessageBox.Show("Success!");
}
catch (Exception e)
{
MakeReport(e.Message);
}
break;
}
case DPFP.Processing.Enrollment.Status.Failed: // report failure and restart capturing
{
Enroller.Clear();
Stop();
UpdateStatus();
OnTemplate(null);
Start();
break;
}
}
}
}

You would need to use parameters
using(var cmd = new MySqlCommand("INSERT INTO mssqltable(varbinarycolumn) VALUES (#binaryValue)", conn))
{
cmd.Parameters.Add("#binaryValue", SqlDbType.VarBinary, 8000).Value = bytes;
cmd.ExecuteNonQuery();
}

How do I check a BLOB field has Data or not?

I am working on asp.net with oracle database. I want to print the image of employee which is saved in an old table. I don't even the data type of image saved in the photo field of that table.
I used Image handlers to print the images from newly created table but when I query on old tables the images is not getting printed.
How do I know that is there any image saved in the table?
If there is an image why it's not getting printed?
I'll show you the code for image handler for both the table (NEW , OLD) Image from newly created table is printing very fine but what's the problem in old one.
Can any give me any suggestions ?
Here is my ImgHandler.ashx code ;
public void ProcessRequest (HttpContext context)
{
OracleDataReader rdr = null;
OracleConnection conn = Conn.getConn();
OracleCommand cmd = null;
string ImgType = context.Request.QueryString["typ"].ToString();
try
{
if (ImgType=="edu")//this is working fine
{
cmd = new OracleCommand("select attachment pic from newtbl where lvldcp_code=" + context.Request.QueryString["dcp"] + "and emp_code="+context.Request.QueryString["emp"], conn);
}
else if (ImgType=="profile")
{
cmd = new OracleCommand("select photo pic from oldtbl where emp_code="+context.Request.QueryString["emp"], conn);
}
conn.Open();
rdr = cmd.ExecuteReader();
while (rdr.Read())
{
context.Response.ContentType = "image/jpg";
context.Response.BinaryWrite((byte[])rdr["pic"]);
}
if (rdr != null)
rdr.Close();
}
catch (Exception ex)
{
}
finally
{
if (conn != null)
conn.Close();
}
}

If your queries are returning a blob field value then you could use the OracleBlob class.
public void ProcessRequest (HttpContext context)
{
OracleDataReader rdr = null;
OracleConnection conn = Conn.getConn();
OracleCommand cmd = null;
string ImgType = context.Request.QueryString["typ"].ToString();
try
{
if (ImgType=="edu")//this is working fine
{
cmd = new OracleCommand("select attachment pic from newtbl where lvldcp_code=" + context.Request.QueryString["dcp"] + "and emp_code="+context.Request.QueryString["emp"], conn);
}
else if (ImgType=="profile")
{
cmd = new OracleCommand("select photo pic from oldtbl where emp_code="+context.Request.QueryString["emp"], conn);
}
Byte[] byteArray = null;
OracleBlob blob;
conn.Open();
rdr = cmd.ExecuteReader();
while (rdr.Read())
{
blob = rdr.GetOracleBlob(0);
byteArray = new Byte[blob.Length];
int i = blob.Read(byteArray, 0, System.Convert.ToInt32(blob.Length));
//clob.Length or i > 0 will show if there are bites in the clob or not
}
if (rdr != null)
rdr.Close();
context.Response.ContentType = "image/jpg";
context.Response.BinaryWrite(byteArray);
}
catch (Exception ex)
{
}
finally
{
if (conn != null)
conn.Close();
}
}

C# ASP.Net Processing Issue - Reading Table While Deleting Rows

Simply, I have an application that has one page that deletes and then re-adds/refreshes the records into a table every 30 seconds. I have another page that runs every 45 seconds that reads the table data and builds a chart.
The problem is, in the read/view page, every once in a while I get a 0 value (from a max count) and the chart shows nothing. I have a feeling that this is happening because the read is being done at the exact same time the delete page has deleted all the records in the table but has not yet refreshed/re-added them.
Is there a way in my application I can hold off on the read when the table is being refreshed?
Best Regards,
Andy
C#
ASP.Net 4.5
SQL Server 2012
My code below is run in an ASP.Net 4.5 built Windows service. It deletes all records in the ActualPlot table and then refreshes/adds new records from a text file every 30 seconds. I basically need to block (lock?) any user from reading the ActualPlot table while the records are being deleted and refreshed. Can you PLEASE help me change my code to do this?
private void timer1_Tick(object sender, ElapsedEventArgs e)
{
// Open the SAP text files, clear the data in the tables and repopulate the new SAP data into the tables.
var cnnString = ConfigurationManager.ConnectionStrings["TaktBoardsConnectionString"].ConnectionString;
SqlConnection conn = new SqlConnection(cnnString);
SqlConnection conndetail = new SqlConnection(cnnString);
SqlConnection connEdit = new SqlConnection(cnnString);
SqlCommand cmdGetProductFile = new SqlCommand();
SqlDataReader reader;
string sql;
// Delete all the records from the ActualPlot and the ActualPlotPreload tables. We are going to repopulate them with the data from the text file.
sql = "DELETE FROM ActualPlotPreload";
try
{
conn.Open();
SqlCommand cmd = new SqlCommand(sql, conn);
cmd.ExecuteNonQuery();
}
catch (System.Data.SqlClient.SqlException ex)
{
string msg = "Delete Error:";
msg += ex.Message;
Library.WriteErrorLog(msg);
}
finally
{
conn.Close();
}
sql = "DELETE FROM ActualPlot";
try
{
conn.Open();
SqlCommand cmd = new SqlCommand(sql, conn);
cmd.ExecuteNonQuery();
}
catch (System.Data.SqlClient.SqlException ex)
{
string msg = "Delete Error:";
msg += ex.Message;
Library.WriteErrorLog(msg);
}
finally
{
conn.Close();
}
// Read the SAP text file and load the data into the ActualPlotPreload table
sql = "SELECT DISTINCT [BoardName], [ProductFile], [ProductFileIdent] FROM [TaktBoards].[dbo].[TaktBoard] ";
sql = sql + "JOIN [TaktBoards].[dbo].[Product] ON [Product].[ProductID] = [TaktBoard].[ProductID]";
cmdGetProductFile.CommandText = sql;
cmdGetProductFile.CommandType = CommandType.Text;
cmdGetProductFile.Connection = conn;
conn.Open();
reader = cmdGetProductFile.ExecuteReader();
string DBProductFile = "";
string DBTischID = "";
string filepath = "";
string[] cellvalues;
DateTime dt, DateCheckNotMidnightShift;
DateTime ldSAPFileLastMod = DateTime.Now;
string MyDateString;
int FileRecordCount = 1;
while (reader.Read())
{
DBProductFile = (string)reader["ProductFile"];
DBTischID = (string)reader["ProductFileIdent"];
filepath = "c:\\inetpub\\wwwroot\\WebApps\\TaktBoard\\FilesFromSAP\\" + DBProductFile;
FileInfo fileInfo = new FileInfo(filepath); // Open file
ldSAPFileLastMod = fileInfo.LastWriteTime; // Get last time modified
try
{
StreamReader sr = new StreamReader(filepath);
FileRecordCount = 1;
// Populate the AcutalPlotPreload table from with the dates from the SAP text file.
sql = "INSERT into ActualPlotPreload (ActualDate, TischID) values (#ActualDate, #TischID)";
while (!sr.EndOfStream)
{
cellvalues = sr.ReadLine().Split(';');
if (FileRecordCount > 1 & cellvalues[7] != "")
{
MyDateString = cellvalues[7];
DateTime ldDateCheck = DateTime.ParseExact(MyDateString, "M/dd/yyyy", null);
DateTime dateNow = DateTime.Now;
string lsDateString = dateNow.Month + "/" + dateNow.Day.ToString("d2") + "/" + dateNow.Year;
DateTime ldCurrentDate = DateTime.ParseExact(lsDateString, "M/dd/yyyy", null);
string lsTischID = cellvalues[119];
if (ldDateCheck == ldCurrentDate)
{
try
{
conndetail.Open();
SqlCommand cmd = new SqlCommand(sql, conndetail);
cmd.Parameters.Add("#ActualDate", SqlDbType.DateTime);
cmd.Parameters.Add("#TischID", SqlDbType.VarChar);
cmd.Parameters["#TischID"].Value = cellvalues[119];
MyDateString = cellvalues[7] + " " + cellvalues[55];
dt = DateTime.ParseExact(MyDateString, "M/dd/yyyy H:mm:ss", null);
cmd.Parameters["#ActualDate"].Value = dt;
// Ignore any midnight shift (12am to 3/4am) units built.
DateCheckNotMidnightShift = DateTime.ParseExact(cellvalues[7] + " 6:00:00", "M/dd/yyyy H:mm:ss", null);
if (dt >= DateCheckNotMidnightShift)
{
cmd.ExecuteNonQuery();
}
}
catch (System.Data.SqlClient.SqlException ex)
{
string msg = "Insert Error:";
msg += ex.Message;
Library.WriteErrorLog(msg);
}
finally
{
conndetail.Close();
}
}
}
FileRecordCount++;
}
sr.Close();
}
catch
{ }
finally
{ }
}
conn.Close();
// Get the unique TischID's and ActualDate from the ActualPlotPreload table. Then loop through each one, adding the ActualUnits
// AcutalDate and TischID to the ActualPlot table. For each unique TischID we make sure that we reset the liTargetUnits to 1 and
// count up as we insert.
SqlCommand cmdGetTischID = new SqlCommand();
SqlDataReader readerTischID;
int liTargetUnits = 0;
string sqlInsert = "INSERT into ActualPlot (ActualUnits, ActualDate, TischID) values (#ActualUnits, #ActualDate, #TischID)";
sql = "SELECT DISTINCT [ActualDate], [TischID] FROM [TaktBoards].[dbo].[ActualPlotPreload] ORDER BY [TischID], [ActualDate] ASC ";
cmdGetTischID.CommandText = sql;
cmdGetTischID.CommandType = CommandType.Text;
cmdGetTischID.Connection = conn;
conn.Open();
readerTischID = cmdGetTischID.ExecuteReader();
DBTischID = "";
DateTime DBActualDate;
string DBTischIDInitial = "";
while (readerTischID.Read())
{
DBTischID = (string)readerTischID["TischID"];
DBActualDate = (DateTime)readerTischID["ActualDate"];
if (DBTischIDInitial != DBTischID)
{
liTargetUnits = 1;
DBTischIDInitial = DBTischID;
}
else
{
liTargetUnits++;
}
try
{
conndetail.Open();
SqlCommand cmd = new SqlCommand(sqlInsert, conndetail);
cmd.Parameters.Add("#ActualUnits", SqlDbType.Real);
cmd.Parameters.Add("#ActualDate", SqlDbType.DateTime);
cmd.Parameters.Add("#TischID", SqlDbType.VarChar);
cmd.Parameters["#TischID"].Value = DBTischID;
cmd.Parameters["#ActualDate"].Value = DBActualDate;
cmd.Parameters["#ActualUnits"].Value = liTargetUnits;
cmd.ExecuteNonQuery();
cmd.Parameters.Clear();
}
catch (System.Data.SqlClient.SqlException ex)
{
string msg = "Insert Error:";
msg += ex.Message;
Library.WriteErrorLog(msg);
}
finally
{
conndetail.Close();
}
}
conn.Close();
Library.WriteErrorLog("SAP text file data has been imported.");
}

If the data is being re-added right back after the delete (basically you know what to re-add before emptying the table), you could have both operation within the same SQL transaction, so that the data will be available to the other page only when it has been re-added.
I mean something like that :
public bool DeleteAndAddData(string connString)
{
using (OleDbConnection conn = new OleDbConnection(connString))
{
OleDbTransaction tran = null;
try
{
conn.Open();
tran = conn.BeginTransaction();
OleDbCommand deleteComm = new OleDbCommand("DELETE FROM Table", conn);
deleteComm.ExecuteNonQuery();
OleDbCommand reAddComm = new OleDbCommand("INSERT INTO Table VALUES(1, 'blabla', 'etc.'", conn);
reAddComm.ExecuteNonQuery();
tran.Commit();
}
catch (Exception ex)
{
tran.Rollback();
return false;
}
}
return true;
}

If your queries don't take too long to execute, you can start the two with a difference of 7.5 seconds, as there is a collision at every 90 seconds when the read/write finishes 3 cycles, and read/view finishes 2 cycles.
That being said, it's not a fool-proof solution, just a trick based on assumptions, in case you wan't to be completely sure that read/view never happens when read/write cycle is happening, try considering having a Read Lock. I would recommend reading Understanding how SQL Server executes a query and Locking in the Database Engine
Hope that helps.

I would try a couple of things:
Make sure your DELETE + INSERT operation is occurring within a single transaction:
BEGIN TRAN
DELETE FROM ...
INSERT INTO ...
COMMIT
If this isn't a busy table, try locking hints your SELECT statement. For example:
SELECT ...
FROM Table
WITH (UPDLOCK, HOLDLOCK)
In the case where the update transactions starts while your SELECT statement is running, this will cause that transaction to wait until the SELECT is finished. Unfortunately it will block other SELECT statements too, but you don't risk reading dirty data.

I was not able to figure this out but I changed my code so the program was not deleting all the rows in the ActualPlot table but checking to see if the row was there and if not adding the new row from the text file.

I want to insert the data to the database

I want insert the data into the database.
string pa = "4898";
string equery="Insert into Users(pas)values('"+pa+"')";
while I inserting the data to the database,its says string or binary data would be truncated the statement has been terminated. So I changed nvarchar(50) into nvarchar(max) in the table.
And these statement has been executed and saved in the database of unreadable format like 傉䝎਍ਚ this. And how can be over come this problem and save the data in the database as "4898".
private void save_Click(object sender, EventArgs e)
{
string password = "4898";
string equery="Insert into Users(name,gender,dateofbirth,age,fathername,
address,citiy,state,country,zipcode,
mobile,phone,email,jobtitle,
dateofjoin,pic,passwords)
values('"+nametxt.Text.ToString().Trim()+"',
#gender,
'"+dateofbirth.Text.ToString().Trim()+"',
'"+age.Value.ToString().Trim()+"',
'"+fathertxt.Text.ToString().Trim()+"',
'"+addresstxt.Text.ToString().Trim()+"',
'"+citiytxt.Text.ToString().Trim()+"',
'"+statetxt.Text.ToString().Trim()+"',
'"+country.Text.ToString().Trim()+"',
'"+ziptxt.Text.ToString().Trim()+"',
'"+mobiletxt.Text.ToString().Trim()+"',
'"+phonetxt.Text.ToString().Trim()+"',
'"+emailtxt.Text.ToString().Trim()+"',
'"+jobtxt.Text.ToString().Trim()+
"','"+dateofjoin.Text.ToString().Trim()+"',
'"+password+"',#pic)";
SqlCommand cmd = new SqlCommand(equery, con);
if(male.Checked)
cmd.Parameters.AddWithValue("#gender","Male");
else
cmd.Parameters.AddWithValue("#gender","Female");
MemoryStream stream = new MemoryStream();
pictureBox1.Image.Save(stream,System.Drawing.Imaging.ImageFormat.Png);
byte[] pic = stream.ToArray();
cmd.Parameters.AddWithValue("#pic", pic);
try
{
con.Open();
cmd.ExecuteNonQuery();
MessageBox.Show("Saved Successfully");
}
catch (Exception ex)
{
MessageBox.Show(ex.Message);
}
finally
{
con.Close();
}
MessageBox.Show("Saved Successfully");
idtxt.Text = "";
nametxt.Text = "";
age.Value = Convert.ToDecimal(null);
male.Checked = false;
female.Checked = false;
dateofbirth.Text = "";
fathertxt.Text = "";
addresstxt.Text = "";
citiytxt.Text = "";
statetxt.Text = "";
country.Text = "";
ziptxt.Text = "";
mobiletxt.Text = "";
phonetxt.Text = "";
emailtxt.Text = "";
dateofjoin.Text = "";
jobtxt.Text = "";
pictureBox1.Image = null;
}

Looking at your insert statement:
"Insert into Users(... pic,passwords) values (... '"+password+"',#pic)";
it seems that you interchanged pic and passwords.
Also, as others already pointed out in the comments, you should use only parameterised queries (to prevent SQL Injection), especially when handling text inserted by an user or any other untrusted source.

There is a Logical error in your code. Interchange
'"+password+"',#pic)";
in your string equery.
By the way a suggestion! It is better to use 'linq to Sql' rather than query pass method! Because 'Linq to Sql' does every thing for you. you just have to give values :)