I want to export all photographs from our database into a data table. I will then loop through the table and save each image to disk. There are approx 7000 photos.
When I start the process I can retrieve around 4000 photographs before I start to get error messages like, Exception of type 'System.OutOfMemoryException' was thrown.
If I change the SQL query to retrieve half as many photographs, say 3500 then the process completes successfully.
While I have now achieved what I wanted by modifying the SQL each time I run the code, I would like to improve my code so that all 7000 photographs are returned. Could somebody please advise on a better process.
Here is my method
public static DataTable GetAllPhotos()
{
DataTable dt = new DataTable();
dt.Columns.Add("personId", typeof(string));
dt.Columns.Add("Photo", typeof(Bitmap));
string SQL = "";
byte[] getImg = new byte[0];
byte[] BitmapImg = new byte[0];
string personId = "";
SqlConnection conn = new SqlConnection();
conn.ConnectionString = _connString;
SQL = #"select per.person_id,pho.photo
from person as per
left join photo as pho on per.photo_id = pho.photo_id
where photo is not null";
conn.Open();
SqlDataReader dr = null;
SqlCommand cmd = new SqlCommand(SQL, conn);
dr = cmd.ExecuteReader();
while (dr.Read())
{
try
{
getImg = (byte[])dr["Photo"];
personId = Convert.ToString(dr["person_id"]);
MemoryStream str = new MemoryStream(getImg);
Bitmap bitmap = new Bitmap(Image.FromStream(str));
BitmapImg = ImageToByte(bitmap);
dt.Rows.Add(personId, bitmap);
}
catch (Exception ex)
{
LogWriter.WriteLine(personId + ex.Message.ToString());
}
}
conn.Close();
return dt;
}
If your intent is saving images to disk, then why would you get them into an intermediate datatable? Wouldn't it be better if you directly write out to disk as you read them? If so, assuming those are .bmp files:
public static void DumpAllPhotos()
{
string sql = #"select per.person_id,pho.photo
from person as per
inner join photo as pho on per.photo_id = pho.photo_id";
string folder = #"c:\MyFolder"; // output folder
using (SqlConnection con = new SqlConnection(_connString))
using (SqlCommand cmd = new SqlCommand(sql,con))
{
con.Open();
var rdr = cmd.ExecuteReader();
while (rdr.Read())
{
var bytes = (byte[])rdr["photo"];
var path = Path.Combine(folder, $"{rdr["person_id"].ToString()}.bmp");
File.WriteAllBytes(path, bytes);
}
con.Close();
}
}
Bitmap has to be disposed, you're using too many handles.
So your while loop should be something like this:
while (dr.Read())
{
try
{
getImg = (byte[])dr["Photograph"];
personId = Convert.ToString(dr["person_id"]);
MemoryStream str = new MemoryStream(getImg);
Bitmap bitmap = new Bitmap(Image.FromStream(str));
BitmapImg = ImageToByte(bitmap);
dt.Rows.Add(personId, bitmap);
bitmap.Dipose(); // <--- DISPOSE!!
}
catch (Exception ex)
{
LogWriter.WriteLine(personId + ex.Message.ToString());
}
}
or maybe even better:
while (dr.Read())
{
try
{
getImg = (byte[])dr["Photograph"];
personId = Convert.ToString(dr["person_id"]);
MemoryStream str = new MemoryStream(getImg);
using (Bitmap bitmap = new Bitmap(Image.FromStream(str))) {
BitmapImg = ImageToByte(bitmap);
dt.Rows.Add(personId, bitmap);
}
}
catch (Exception ex)
{
LogWriter.WriteLine(personId + ex.Message.ToString());
}
}
Related
I have a file with 9 million to 13 million records. As the list has a limit size I have broken the code into chunks of 1 million records and trying to insert the record.
Here is code to read and break the file in chunks:
public static void InsertBulkData(string file)
{
List<FileData> lstFileData = new List<FileData>();
using (FileStream fs = File.Open(file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
using (BufferedStream bs = new BufferedStream(fs))
{
using (StreamReader sr = new StreamReader(bs))
{
string line = string.Empty;
while ((line = sr.ReadLine()) != null)
{
FileData obj = new FileData();
obj.Property1= line.Substring(5, 9).Trim();
obj.Property2= line.Substring(19, 40).Trim();
lstFileData.Add(obj);
if (lstFileData.Count == 1000000)
{
InsertDataIntoDatabase(lstFileData);
lstFileData = null;
GC.Collect();
lstFileData = new List<FileData>();
}
}
}
}
}
}
Now we are getting in the loop 9 times. But when i comes to saving of data it saves for the first time only. rest 8 times it throws exception.
private static void InsertDataIntoDatabase(List<FileData> lstFileData)
{
String query = #"INSERT INTO table
(
PrimaryCol,
Column1,
Column2
)
VALUES
(
primaryCol.NEXTVAL,
:Property1,
:Property2,
)";
using (OracleConnection Conn = new OracleConnection())
{
try
{
Conn.ConnectionString = ConfigurationManager.ConnectionStrings["connection"].ToString();
Conn.Open();
using (var command = Conn.CreateCommand())
{
command.CommandText = query;
command.CommandType = CommandType.Text;
command.BindByName = true;
// In order to use ArrayBinding, the ArrayBindCount property
// of OracleCommand object must be set to the number of records to be inserted
command.ArrayBindCount = lstFileData.Count;
command.Parameters.Add(":Property1", OracleDbType.Varchar2, lstFileData.Select(c => c.Property1).ToArray(), ParameterDirection.Input);
command.Parameters.Add(":Property2", OracleDbType.Varchar2, lstFileData.Select(c => c.Property2).ToArray(), ParameterDirection.Input);
command.ExecuteNonQuery();
Conn.Close();
}
}
catch (Exception ex)
{
Conn.Close();
Console.WriteLine("Exception is:" + ex.InnerException);
}
}
}
So After first iteration we getting error in command.ExecuteNonQuery(). Can any one tell why and what is the possible solution?
Your problem comes from the use of a buffer that reaches its maximum size. Moreover, if you use successive inserts, you will have performance problems. You should use another method (with bulk copy). Here is a sample code:
public static void BulkCopy(string connectionString, DataTable dataTableToSend, string SQLTableName)
{
SqlConnection connexion = null;
try
{
connexion = new SqlConnection(connectionString);
BulkCopy(connexion, dataTableToSend, SQLTableName);
}
catch (Exception e)
{
throw;
}
finally
{
connexion?.Close();
connexion?.Dispose();
}
}
int Modulo=0;
//Create a datatable MyDataTable with same structure of your table in database
// you can do : select * from MyTable where 1<>1 with command for get structured dataset (and datatable)
using (System.IO.StreamReader file = new System.IO.StreamReader(fileName, Encoding.UTF8))
{
while ((Line = file.ReadLine()) != null)
{
counter++;
DataRow newrow = MyDataTable.NewRow();
//Set value
newrow["I_ID"] = counter;
newrow["L_NomFichier"] = FileObj.Name;
newrow["D_CreationFichier"] = FileObj.LastWriteTime;
newrow["L_Ligne"] = Line.DBNullIfNullOrEmpty();
newrow["D_DATCRE"] = newrow["D_DATMAJ"] = DateTime.Now;
newrow["C_UTICRE"] = newrow["C_UTIMAJ"] = GlobalParam.User;
MyDataTable.Rows.Add(newrow);
if (Modulo % 1000)
{
SqlHelper.BulkCopy(DBAccess.CONN_STRING, MyDataTable, "YOURTABLENAME");
MyDataTable.Rows.Clear();
}
}
SqlHelper.BulkCopy(DBAccess.CONN_STRING, MyDataTable, "YOURTABLENAME");
MyDataTable.Rows.Clear();
file.Close();
}
When I'm trying to retrieve my image by combo box there show the massage . Parameter not valid
I was try many way but problem is same ..
Every time I run the code below, I get same massage.
private void cBoxSearch_SelectedIndexChanged(object sender, EventArgs e)
{
try
{
con = ConnectionController.GetInstance().GetConnection();
con.Open();
com = new SQLiteCommand("SELECT * FROM Stock WHERE ProductName = '" + cBoxSearch.Text + "' ", con);
myReader = com.ExecuteReader();
product prod = new product();
while (myReader.Read())
{
prod.proid = myReader[0].ToString();
prod.prodname = myReader[1].ToString();
prod.proMdl = myReader[2].ToString();
prod.serialN = myReader[3].ToString();
prod.byibgPr = myReader[4].ToString();
prod.sellPr = myReader[5].ToString();
prod.quantity = myReader[6].ToString();
tbxProductID.Text = prod.proid;
tbxName.Text =prod.prodname;
tbxModel.Text = prod.proMdl;
tbxserial.Text = prod.serialN;
tbxbyingprice.Text = prod.byibgPr;
tbxSellingprice.Text = prod.sellPr;
tbxQuantity.Text = prod.quantity;
prod.imgg = (byte[])(myReader[7] );
if (prod.imgg == null)
{
pBX.Image = null;
}
else
{
MemoryStream mstrm = new MemoryStream(prod.imgg);
Bitmap bmp = new Bitmap(mstrm);
}
}
con.Close();
}
catch (Exception ex)
{
MessageBox.Show(ex.Message);
com.Cancel();
con.Close();
}
Try to consult this question:
convert binary to bitmap using memory stream
Also, it might depend on DBType of the 8th column of the query.
I am trying to save and retrieve a file to and from a database. I am serializing the object ans saving it as binary. However, when trying to deserialize I get the error that the input stream is not a valid binary format. I tried several solutions and this is what I've put together so far:
public void saveFile(string filename, string file, object o)
{
byte[] myFile;
if (o != null)
{
BinaryFormatter bf = new BinaryFormatter();
using (MemoryStream ms = new MemoryStream())
{
bf.Serialize(ms, o);
myFile= ms.ToArray();
}
String insert = "INSERT INTO user_files(FileName, Username, File) VALUES ('myfile','noname','"+ myFile + "')";
MySqlCommand command = new MySqlCommand(insert, connection);
try
{
connection.Open();
command.ExecuteNonQuery();
}
catch
{
MessageBox.Show("Sorry, something went wrong");
}
finally
{ connection.Close(); }
}
And here is the Load
public TrafficMonitor LoadFile(string user, string filename)
{
TrafficMonitor obj = null;
byte[] myFile = null;
DataTable dt = new DataTable();
MySqlDataAdapter getCommand = new MySqlDataAdapter("Select File from user_files where Username='noname' and filename='myfile'" , connection);
try
{
connection.Open();
getCommand.Fill(dt);
foreach (DataRow row in dt.Rows)
{
myFile= (byte[])row["File"];
}
MemoryStream memStream = new MemoryStream();
BinaryFormatter binForm = new BinaryFormatter();
memStream.Write(myFile, 0, myFile.Length);
memStream.Seek(0, SeekOrigin.Begin);
obj = (TrafficMonitor)binForm.Deserialize(memStream);
}
catch { MessageBox.Show("Sorry, something went wrong"); }
finally { connection.Close(); }
return obj;
}
Here is how I inserted binary data to MySQL:
byte[] imgBuffer = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 };
MySqlCommand query = new MySqlCommand();
query.Connection = _connection;
query.CommandText = "INSERT INTO Photos(id,img) VALUES(?id,?img)";
var id = Guid.NewGuid();
query.Parameters.Add("?id", MySqlDbType.Binary).Value = id.ToByteArray();
query.Parameters.Add("?img", MySqlDbType.Blob).Value = imgBuffer;
query.ExecuteNonQuery();
And how I read binary data from MySQL:
MySqlCommand query = new MySqlCommand();
query.Connection = _connection;
query.CommandText = "SELECT * FROM Photos WHERE id=#ID";
query.Parameters.Add("#id", MySqlDbType.Binary).Value = guid.ToByteArray();
using (MySqlDataReader reader = query.ExecuteReader())
{
if (reader.Read())
{
Guid id = reader.GetGuid(0);
byte[] imgBuffer = (byte[])reader.GetValue(1);
}
}
I am working on asp.net with oracle database. I want to print the image of employee which is saved in an old table. I don't even the data type of image saved in the photo field of that table.
I used Image handlers to print the images from newly created table but when I query on old tables the images is not getting printed.
How do I know that is there any image saved in the table?
If there is an image why it's not getting printed?
I'll show you the code for image handler for both the table (NEW , OLD) Image from newly created table is printing very fine but what's the problem in old one.
Can any give me any suggestions ?
Here is my ImgHandler.ashx code ;
public void ProcessRequest (HttpContext context)
{
OracleDataReader rdr = null;
OracleConnection conn = Conn.getConn();
OracleCommand cmd = null;
string ImgType = context.Request.QueryString["typ"].ToString();
try
{
if (ImgType=="edu")//this is working fine
{
cmd = new OracleCommand("select attachment pic from newtbl where lvldcp_code=" + context.Request.QueryString["dcp"] + "and emp_code="+context.Request.QueryString["emp"], conn);
}
else if (ImgType=="profile")
{
cmd = new OracleCommand("select photo pic from oldtbl where emp_code="+context.Request.QueryString["emp"], conn);
}
conn.Open();
rdr = cmd.ExecuteReader();
while (rdr.Read())
{
context.Response.ContentType = "image/jpg";
context.Response.BinaryWrite((byte[])rdr["pic"]);
}
if (rdr != null)
rdr.Close();
}
catch (Exception ex)
{
}
finally
{
if (conn != null)
conn.Close();
}
}
If your queries are returning a blob field value then you could use the OracleBlob class.
public void ProcessRequest (HttpContext context)
{
OracleDataReader rdr = null;
OracleConnection conn = Conn.getConn();
OracleCommand cmd = null;
string ImgType = context.Request.QueryString["typ"].ToString();
try
{
if (ImgType=="edu")//this is working fine
{
cmd = new OracleCommand("select attachment pic from newtbl where lvldcp_code=" + context.Request.QueryString["dcp"] + "and emp_code="+context.Request.QueryString["emp"], conn);
}
else if (ImgType=="profile")
{
cmd = new OracleCommand("select photo pic from oldtbl where emp_code="+context.Request.QueryString["emp"], conn);
}
Byte[] byteArray = null;
OracleBlob blob;
conn.Open();
rdr = cmd.ExecuteReader();
while (rdr.Read())
{
blob = rdr.GetOracleBlob(0);
byteArray = new Byte[blob.Length];
int i = blob.Read(byteArray, 0, System.Convert.ToInt32(blob.Length));
//clob.Length or i > 0 will show if there are bites in the clob or not
}
if (rdr != null)
rdr.Close();
context.Response.ContentType = "image/jpg";
context.Response.BinaryWrite(byteArray);
}
catch (Exception ex)
{
}
finally
{
if (conn != null)
conn.Close();
}
}
private void view_Load(object sender, EventArgs e)
{
try
{
SqlConnection con = new SqlConnection();
con.ConnectionString = "Data Source=SOFT;Initial Catalog=Dev01;Integrated Security=True";
con.Open();
//Retrieve BLOB from database into DataSet.
SqlDataReader myReader = null;
SqlCommand cmd = new SqlCommand("select * from empdetails", con);
myReader = cmd.ExecuteReader();
while (myReader.Read())
{
lbl_fname.Text = myReader["firstname"].ToString();
lbl_mname.Text = myReader["middlename"].ToString();
lbl_lname.Text = myReader["lastname"].ToString();
lbl_gender.Text = myReader["gender"].ToString();
lbl_dob.Text = myReader["dob"].ToString();
lbl_qualification.Text = myReader["qualification"].ToString();
lbl_skills.Text = myReader["skills"].ToString();
lbl_userid.Text = myReader["username"].ToString();
lbl_pwd.Text = myReader["password"].ToString();
lbl_cpwd.Text = myReader["confirmpassword"].ToString();
lbl_mno.Text = myReader["mobilenumber"].ToString();
lbl_altmno.Text = myReader["alternativenumber"].ToString();
lbl_email.Text = myReader["email"].ToString();
lbl_presentadd.Text = myReader["presentaddress"].ToString();
lbl_permanentadd.Text = myReader["permanentaddress"].ToString();
}
myReader.Close();
SqlDataAdapter da = new SqlDataAdapter(cmd);
DataSet ds = new DataSet();
da.Fill(ds, "empdetails");
int c = ds.Tables["empdetails"].Rows.Count;
if (c > 0)
{
//BLOB is read into Byte array, then used to construct MemoryStream,
//then passed to PictureBox.
//SqlCommand cmd1=new SqlCommand("Select photo from empdetails");
Byte[] bytedata = new Byte[0];
bytedata = (Byte[])(ds.Tables["empdetails"].Rows[c - 1]["photo"]);
MemoryStream ms = new MemoryStream(bytedata);
pictureBox1.Image = Image.FromStream(ms,true); //HERE I AM GETTING ERROR
}
con.Close();
}
catch (Exception ex)
{
MessageBox.Show(ex.Message);
}
}
I already checked various pages but that didn't solve my problem. I am getting an error only in this part:
"pictureBox1.Image = Image.FromStream(ms,true);"
The data type of your column should be image try
public byte[] ConvertImageToByte(Image image, ImageFormat format)
{
var stream = new MemoryStream();
image.Save(stream, format);
return stream.ToArray();
}
public Image ConvertBytesToImage(object _image)
{
byte[] byteImage = (byte[])(_image);
MemoryStream ms = new MemoryStream(byteImage);
ms.Write(byteImage, 0, byteImage.Length);
Image img = Image.FromStream(ms);
return img;
}
Saving the byte in your database
OpenFileDialog file = new OpenFileDialog();
file.Title = "Please Select the Employee Photo (Dimension 128x128)";
file.Filter = "JPEG Files|*.jpg|Bitmap Files|*.bmp|Gif Files|*.gif|PNG Files|*.png";
file.DefaultExt = "jpg";
file.ShowDialog();
if (!string.IsNullOrEmpty(file.FileName))
{
byte[] bytePhoto = ConvertImageToBytes(Image.FromFile(file.FileName), ImageFormat.Png);
//bytePhoto is the object to save in your database
}
Retrieve the byte and display as image
byte[] bytePhoto = (byte[])ds.Tables["empdetails"].Rows[0]["Photo"];
pictureBox1.Image = ConvertBytesToImage(bytePhoto);