I have a Table Blob, which has a varbinary(max) as a column. Now I want to store data into the database using a Filestream. The data can be really big (in my case 1.5GB) so I dont want to load the whole data into a buffer.
What I tried:
using (FileStream fs = File.Open(#"BigData.iso", FileMode.Open))
{
using (SqlConnection conn = new SqlConnection())
{
conn.ConnectionString = #"...";
conn.Open();
SqlCommand command = new SqlCommand("INSERT INTO Blob Values (#0, #1)", conn);
command.Parameters.Add(new SqlParameter("0", Guid.NewGuid()));
var sqlb = new SqlBytes(fs);
command.Parameters.Add(new SqlParameter("1", SqlDbType.VarBinary, -1)).Value = sqlb;
command.ExecuteNonQuery();
}
}
But I got an OutOfMemoryException, because SqlBytes initialze its buffer to the whole size of the data.
I know there is a FILESTREAM feature from Microsoft, but I don't want to use it.
Is there a way to achieve this?
You can read the file in small chunks and append them to the data column.
You will need an IDENTITY column or another column(s) that can be used as a key to execute UPDATE statements. Here is an example using an IDENTITY column:
Create a table to store the data
CREATE TABLE [dbo].[table1](
[ID] [int] IDENTITY(1,1) PRIMARY KEY NOT NULL,
[Data] [varbinary](max) NULL,
)
Implement C# to insert/update data in chunks
private const string C_SqlConnectionString = #"Server=SERVERNAME;Database=DBNAME;Trusted_Connection=yes;";
private const int C_FileChunkSizeBytes = 1024 * 1024; // 1 MB
private static void storeFile(string filepath)
{
using (FileStream fs = File.Open(filepath, FileMode.Open))
{
using (SqlConnection conn = new SqlConnection())
{
conn.ConnectionString = C_SqlConnectionString;
conn.Open();
// Use a transaction to ensure that all parts of the file get stored to DB
SqlCommand command = new SqlCommand("BEGIN TRAN", conn);
command.ExecuteNonQuery();
var pos = 0;
byte[] fileBytes = null;
int sqlRowId = 0;
// Read the file in chunks
while (pos < fs.Length)
{
// Read file bytes
var bytesToRead = pos + C_FileChunkSizeBytes < fs.Length
? C_FileChunkSizeBytes
: (int)(fs.Length - pos);
fileBytes = new byte[bytesToRead];
fs.Read(fileBytes, 0, bytesToRead);
// Store bytes to a parameter
var varbinary = new SqlParameter("0", System.Data.SqlDbType.VarBinary, -1);
varbinary.Value = fileBytes;
if (pos == 0)
{
// If this is the first chunk, then we need to INSERT
// The HOLDLOCK hint will hold a lock on the table until transaction completes (or is rolled back)
command = new SqlCommand("INSERT INTO [dbo].[table1] WITH(HOLDLOCK) VALUES(#0)", conn);
command.Parameters.Add(varbinary);
command.ExecuteNonQuery();
// Get the row ID for the inserted row
command = new SqlCommand("SELECT ##IDENTITY", conn);
sqlRowId = Convert.ToInt32(command.ExecuteScalar());
}
else
{
// Update existing row and append data
command = new SqlCommand("UPDATE [dbo].[table1] SET [Data] = [Data] + #0 WHERE [ID] = #1", conn);
command.Parameters.Add(varbinary);
command.Parameters.Add(new SqlParameter("1", System.Data.SqlDbType.Int)).Value = sqlRowId;
command.ExecuteNonQuery();
}
// ** Good place for a breakpoint
pos += bytesToRead;
}
// Commit transaction
command = new SqlCommand("COMMIT TRAN", conn);
command.ExecuteNonQuery();
conn.Close();
}
}
}
Testing
Place a breakpoint in C# code at the bottom of the while loop, eg at pos += bytesToRead;.
Whilst executing the code, when code execution stops at the breakpoint, check the data in SQL:
SELECT *
,LEN([Data]) AS [Length]
FROM [dbo].[table1] WITH(NOLOCK)
The NOLOCK hint will let us see data in uncommitted transactions. LEN([Data]) will show how field length grows after each iteration of the while loop.
Related
I am inserting docx file into an SQL table like ,
DECLARE #file AS VARBINARY(MAX)
SELECT #file = CAST(bulkcolumn AS VARBINARY(MAX))
FROM OPENROWSET(
BULK
'D:\abc.docx',
SINGLE_BLOB ) AS x
INSERT INTO ItemData(Id,fileData)
SELECT NEWID(),#file
Now file inserted into table properly.
Read File
For reading file, I am using bellow code :
"ItemNumber" is used for identity.
Everything is working fine.
But I have to show that file to the Document Viewer in WPF.
string cmdString = "Select fileData.PathName() As Path from ItemData where ItemNumber = '" + itemNumber + "' ";
SqlCommand cmd = new SqlCommand(cmdString, conn);
Object pathObj = cmd.ExecuteScalar();
if (DBNull.Value != pathObj)
filePath = (string)pathObj;
else
{
throw new System.Exception("fileData.PathName() failed to read the path name for the Chart column.");
}
this.wbControl.Navigate(filePath);
SqlTransaction transaction = conn.BeginTransaction("mainTranaction");
cmd.Transaction = transaction;
cmd.CommandText = "SELECT GET_FILESTREAM_TRANSACTION_CONTEXT()";
Object obj = cmd.ExecuteScalar();
byte[] txContext = (byte[])obj;
SqlFileStream sqlFileStream = new SqlFileStream(filePath, txContext, FileAccess.ReadWrite);
//byte[] buffer2 = new byte[(int)sqlFileStream.Length];
byte[] buffer = new byte[512];
int numBytes = 0;
string someData = "EKG data.";
Encoding unicode = Encoding.GetEncoding(0);
sqlFileStream.Write(unicode.GetBytes(someData.ToCharArray()), 0, someData.Length);
sqlFileStream.Seek(0L, SeekOrigin.Begin);
numBytes = sqlFileStream.Read(buffer, 0, buffer.Length);
string readData = unicode.GetString(buffer);
if (numBytes != 0)
Console.WriteLine(readData);
//Because reading and writing are finished, FILESTREAM
//must be closed. This closes the c# FileStream class,
//but does not necessarily close the the underlying
//FILESTREAM handle.
sqlFileStream.Close();
//The final step is to commit or roll back the read and write
//operations that were performed on the FILESTREAM BLOB.
cmd.Transaction.Commit();
How to I display SqlFileStream to Document Viewer in WPF?
I am storing docx file in database.
I searched on the net something but nothing really helped me. I want to update, with a list of article, a database, but the way that I've found is really slow.
This is my code:
List<Article> costs = GetIdCosts(); //here there are 70.000 articles
conn = new OleDbConnection(string.Format(MDB_CONNECTION_STRING, PATH, PSW));
conn.Open();
transaction = conn.BeginTransaction();
using (var cmd = conn.CreateCommand())
{
cmd.Transaction = transaction;
cmd.CommandText = "UPDATE TABLE_RO SET TABLE_RO.COST = ? WHERE TABLE_RO.ID = ?;";
for (int i = 0; i < costs.Count; i++)
{
double cost = costs[i].Cost;
int id = costs[i].Id;
cmd.Parameters.AddWithValue("data", cost);
cmd.Parameters.AddWithValue("id", id);
if (cmd.ExecuteNonQuery() != 1) throw new Exception();
}
}
transaction.Commit();
But this way take a lot of minutes something like 10 minutes or more. There are another way to speed up this updating ? Thanks.
Try modifying your code to this:
List<Article> costs = GetIdCosts(); //here there are 70.000 articles
// Setup and open the database connection
conn = new OleDbConnection(string.Format(MDB_CONNECTION_STRING, PATH, PSW));
conn.Open();
// Setup a command
OleDbCommand cmd = new OleDbCommand();
cmd.Connection = conn;
cmd.CommandText = "UPDATE TABLE_RO SET TABLE_RO.COST = ? WHERE TABLE_RO.ID = ?;";
// Setup the paramaters and prepare the command to be executed
cmd.Parameters.Add("?", OleDbType.Currency, 255);
cmd.Parameters.Add("?", OleDbType.Integer, 8); // Assuming you ID is never longer than 8 digits
cmd.Prepare();
OleDbTransaction transaction = conn.BeginTransaction();
cmd.Transaction = transaction;
// Start the loop
for (int i = 0; i < costs.Count; i++)
{
cmd.Parameters[0].Value = costs[i].Cost;
cmd.Parameters[1].Value = costs[i].Id;
try
{
cmd.ExecuteNonQuery();
}
catch (Exception ex)
{
// handle any exception here
}
}
transaction.Commit();
conn.Close();
The cmd.Prepare method will speed things up since it creates a compiled version of the command on the data source.
Small change option:
Using StringBuilder and string.Format construct one big command text.
var sb = new StringBuilder();
for(....){
sb.AppendLine(string.Format("UPDATE TABLE_RO SET TABLE_RO.COST = '{0}' WHERE TABLE_RO.ID = '{1}';",cost, id));
}
Even faster option:
As in first example construct a sql but this time make it look (in result) like:
-- declaring table variable
declare table #data (id int primary key, cost decimal(10,8))
-- insert union selected variables into the table
insert into #data
select 1121 as id, 10.23 as cost
union select 1122 as id, 58.43 as cost
union select ...
-- update TABLE_RO using update join syntax where inner join data
-- and copy value from column in #data to column in TABLE_RO
update dest
set dest.cost = source.cost
from TABLE_RO dest
inner join #data source on dest.id = source.id
This is the fastest you can get without using bulk inserts.
Performing mass-updates with Ado.net and OleDb is painfully slow. If possible, you could consider performing the update via DAO. Just add the reference to the DAO-Library (COM-Object) and use something like the following code (caution -> untested):
// Import Reference to "Microsoft DAO 3.6 Object Library" (COM)
string TargetDBPath = "insert Path to .mdb file here";
DAO.DBEngine dbEngine = new DAO.DBEngine();
DAO.Database daodb = dbEngine.OpenDatabase(TargetDBPath, false, false, "MS Access;pwd="+"insert your db password here (if you have any)");
DAO.Recordset rs = daodb.OpenRecordset("insert target Table name here", DAO.RecordsetTypeEnum.dbOpenDynaset);
if (rs.RecordCount > 0)
{
rs.MoveFirst();
while (!rs.EOF)
{
// Load id of row
int rowid = rs.Fields["Id"].Value;
// Iterate List to find entry with matching ID
for (int i = 0; i < costs.Count; i++)
{
double cost = costs[i].Cost;
int id = costs[i].Id;
if (rowid == id)
{
// Save changed values
rs.Edit();
rs.Fields["Id"].Value = cost;
rs.Update();
}
}
rs.MoveNext();
}
}
rs.Close();
Note the fact that we are doing a full table scan here. But, unless the total number of records in the table is many orders of magnitude bigger than the number of updated records, it should still outperform the Ado.net approach significantly...
I'm developing an app in C# with .NET and MySQL database. I need to be able to insert and retrieve images in and out of the database and I have a column named 'Image' of type LONGBLOB for that purpose. The insertion goes well but when I try to retrieve the blob the following error pops up:
GetBytes() can only be called on binary or GUID columns
I'm using the following code to select from the database:
Conn.Open();
string sql = #"SELECT `ID`, `Image`, `Note`"
+ " FROM `Item`"
+ " WHERE `ID` = ?ID";
MySqlCommand cmd = new MySqlCommand(sql, Conn);
cmd.Parameters.Add(new MySqlParameter("?ID", iD));
cmd.Prepare();
rdr = cmd.ExecuteReader(CommandBehavior.SequentialAccess);
while (rdr.Read())
{
this.ID= rdr.GetString("ID");
if (!rdr.IsDBNull(1))
{
long len = rdr.GetBytes(1, 0, null, 0, 0);
byte[] ImageBytes = new byte[len];
rdr.GetBytes(1, 0, ImageBytes, 0, (int)len);
MemoryStream ms = new MemoryStream(ImageBytes);
this.Image = Image.FromStream(ms);
}
this.Note = rdr.GetString("Note");
Despite changing the column type into binary and varbinary, I still got the same error.
Does anyone know what I'm doing wrong here?
TIA
Can't you just cast rdr["Image"] as byte[]?
I'm using the following code to insert into a blob field:
MySql.Data.MySqlClient.MySqlConnection conn;
MySql.Data.MySqlClient.MySqlCommand cmd;
conn = new MySql.Data.MySqlClient.MySqlConnection();
cmd = new MySql.Data.MySqlClient.MySqlCommand();
string SQL;
int FileSize;
byte[] rawData;
FileStream fs;
conn.ConnectionString = "server=192.168.1.104;uid=root;" +
"pwd=root;database=cady234;";
fs = new FileStream(#"d:\Untitled.gif", FileMode.Open, FileAccess.Read);
FileSize = (int)fs.Length;
rawData = new byte[FileSize];
fs.Read(rawData, 0, FileSize);
fs.Close();
conn.Open();
string strFileName = "test name";
SQL = "INSERT INTO file (file_name, file_size, file) VALUES ('" + strFileName + "', "+FileSize+", '"+rawData+"')";
cmd.Connection = conn;
cmd.CommandText = SQL;
cmd.ExecuteNonQuery();
conn.Close();
The insertion is ok but the image is not getting displayed while using "Open value in viewer":
The binary data isn't being properly passed to your insert as you are using string concatenation - you'll get rawData.ToString() which probably just prints out the TypeName (hence your binary data being 13 bytes in length compared to the filesize of > 3000 bytes); try this instead:
byte[] rawData = File.ReadAllBytes(#"d:\Untitled.gif");
FileInfo info = new FileInfo(#"d:\Untitled.gif");
int fileSize = Convert.ToInt32(info.Length);
using(MySqlConnection connection = new MySqlConnection("server=192.168.1.104;uid=root;pwd=root;database=cady234;"))
{
using(MySqlCommand command = new MySqlCommand())
{
command.Connection = connection;
command.CommandText = "INSERT INTO file (file_name, file_size, file) VALUES (?fileName, ?fileSize, ?rawData);";
MySqlParameter fileNameParameter = new MySqlParameter("?fileName", MySqlDbType.VarChar, 256);
MySqlParameter fileSizeParameter = new MySqlParameter("?fileSize", MySqlDbType.Int32, 11);
MySqlParameter fileContentParameter = new MySqlParameter("?rawData", MySqlDbType.Blob, rawData.Length);
fileNameParameter.Value = "test name";
fileSizeParameter.Value = fileSize;
fileContentParameter.Value = rawData;
command.Parameters.Add(fileNameParameter);
command.Parameters.Add(fileSizeParameter);
command.Parameters.Add(fileContentParameter);
connection.Open();
command.ExecuteNonQuery();
}
}
I've introduced several concepts for you here; firstly, if you are going to load all of the binary data at once, simply use the static method File.ReadAllBytes - it's a lot less code.
Secondly, there is no need to use the fully qualified namespace each time - use the using directive
Thirdly, (slightly confusingly) there is a also a using statement in C#. This ensures that any object that implements IDisposable is properly cleaned up after itself. In the case of the connection, it will explicitly call Close and Dispose if you command succeeds or fails.
Finally, I've parameterized your query. Parameters are useful for many reasons; they help protect against SQL Injection, and, in this instance, they should also ensure that your data types are handled correctly. You can read more about SqlParameter (which, like, MySqlParameter is a database specific implementation but uses the same principles).
Tested as working with MySQL 5.5.15, MySQL Connector 5.2.7 running under .Net 4
How about this:
It works fine for me.
Exepte I found out that it is the wrong one for me.
<?php
// csv invoeren naar pos.
$CSV = "uitvoer.csv";
// The CSV file has only 2 colums; The "Reference" and the image name (in my case the barcode with "thumb_" in front of it.
$username = "Username";
$password = "Passwwoorrdd";
$database = "POS";
$counter = 0;
// Create connection
$conn = new mysqli("localhost", $username, $password, $database);
// Check connection
if ($conn->connect_error) {
die("Connection failed: " . $conn->connect_error);
}
echo "Connected successfully <br>";
//$mysqli->select_db();
ini_set('max_execution_time', 1000); //300 seconds = 5 minutes
if (($handle = fopen($CSV, "r")) !== FALSE) {
while (($data = fgetcsv($handle, 1050, ",")) !== FALSE) {
// this loops through each line of your csv, putting the values into array elements
$counter++;
$IDEE = $data[0];
// It seems that after opening the image the $data is mixed up.
$imag = "photos/".$data[1]; // where "photos/' the folder is where this php file gets executed (mostly in /var/www/ of /var/www/html/)
$fh = fopen($imag, "r");
$data = addslashes(fread($fh, filesize($imag)));
fclose($fh);
echo " Ref: ".$IDEE." ----".$counter."----<br>";
// If there will be a time-out. You could erase the part what is already done minus 1.
$sql = "UPDATE PRODUCTS SET IMAGE='".$data."' WHERE CODE=$IDEE";
// The table PRODUCTS with IMAGE. There are more data in that table. But i need only to update the IMAGE. The rest is already inserted.
if ($conn->query($sql) === TRUE) {
echo "Tabel <b>products</b> updated successfully<br>";
} else {
echo "<br>Error updating tabel <b>Products</b>: " . $conn->error;
Exit();
}
}
fclose($handle);
}
?>
I have to persist a .csv in my database, but for a more testable application I prefer don't use procedures.
Basically I just generate a file and the next instruction is put this in database.
Someone have some clue about best way to do this in code?
Here is an example to insert blob data in oracle using c# and procedures (you said prefer that means you may).
using System;
using System.Data;
using Oracle.DataAccess.Client;
using Oracle.DataAccess.Types;
using System.IO;
using System.Text;
//Step 1
// Connect to database
// Note: Modify User Id, Password, Data Source as per your database setup
string constr = "User Id=Scott;Password=tiger;Data Source=orcl9i";
OracleConnection con = new OracleConnection(constr);
con.Open();
Console.WriteLine("Connected to database!");
// Step 2
// Note: Modify the Source and Destination location
// of the image as per your machine settings
String SourceLoc = "D:/Images/photo.jpg";
String DestinationLoc = "D:/Images/TestImage.jpg";
// provide read access to the file
FileStream fs = new FileStream(SourceLoc, FileMode.Open,FileAccess.Read);
// Create a byte array of file stream length
byte[] ImageData = new byte[fs.Length];
//Read block of bytes from stream into the byte array
fs.Read(ImageData,0,System.Convert.ToInt32(fs.Length));
//Close the File Stream
fs.Close();
// Step 3
// Create Anonymous PL/SQL block string
String block = " BEGIN " +
" INSERT INTO testblob (id, photo) VALUES (100, :1); " +
" SELECT photo into :2 from testblob WHERE id = 100; " +
" END; ";
// Set command to create Anonymous PL/SQL Block
OracleCommand cmd = new OracleCommand();
cmd.CommandText = block;
cmd.Connection = con;
// Since executing an anonymous PL/SQL block, setting the command type
// as Text instead of StoredProcedure
cmd.CommandType = CommandType.Text;
// Step 4
// Setting Oracle parameters
// Bind the parameter as OracleDbType.Blob to command for inserting image
OracleParameter param = cmd.Parameters.Add("blobtodb", OracleDbType.Blob);
param.Direction = ParameterDirection.Input;
// Assign Byte Array to Oracle Parameter
param.Value = ImageData;
// Bind the parameter as OracleDbType.Blob to command for retrieving the image
OracleParameter param2 = cmd.Parameters.Add("blobfromdb", OracleDbType.Blob);
param2.Direction = ParameterDirection.Output;
// Step 5
// Execute the Anonymous PL/SQL Block
// The anonymous PL/SQL block inserts the image to the
// database and then retrieves the images as an output parameter
cmd.ExecuteNonQuery();
Console.WriteLine("Image file inserted to database from " + SourceLoc);
// Step 6
// Save the retrieved image to the DestinationLoc in the file system
// Create a byte array
byte[] byteData = new byte[0];
// fetch the value of Oracle parameter into the byte array
byteData = (byte[])((OracleBlob)(cmd.Parameters[1].Value)).Value;
// get the length of the byte array
int ArraySize = new int();
ArraySize = byteData.GetUpperBound(0);
// Write the Blob data fetched from database to the filesystem at the
// destination location
FileStream fs1 = new FileStream(#DestinationLoc,
FileMode.OpenOrCreate, FileAccess.Write);
fs1.Write(byteData, 0,ArraySize);
fs1.Close();
Console.WriteLine("Image saved to " + DestinationLoc + " successfully !");
Console.WriteLine("");
Console.WriteLine("***********************************************************");
Console.WriteLine("Before running this application again, execute 'Listing 1' ");
private void btnSave_Click(object sender, EventArgs e)
{
try
{
//Read Image Bytes into a byte array
byte[] blob = ReadFile(txtPath.Text);
//Initialize Oracle Server Connection
con = new OracleConnection(conString);
//Set insert query
string qry = "insert into Imgpn (imgpath,photo) values('" + txtPath.Text + "'," + " :BlobParameter )";
OracleParameter blobParameter = new OracleParameter();
blobParameter.OracleType = OracleType.Blob;
blobParameter.ParameterName = "BlobParameter";
blobParameter.Value = blob;
//Initialize OracleCommand object for insert.
cmd = new OracleCommand(qry, con);
//We are passing Name and Blob byte data as Oracle parameters.
cmd.Parameters.Add(blobParameter);
//Open connection and execute insert query.
con.Open();
cmd.ExecuteNonQuery();
MessageBox.Show("Image added to blob field");
GetImagesFromDatabase();
cmd.Dispose();
con.Close();
//this.Close();
}
catch (Exception ex)
{
MessageBox.Show(ex.ToString());
}
}
byte[] ReadFile(string sPath)
{
//Initialize byte array with a null value initially.
byte[] data = null;
//Use FileInfo object to get file size.
FileInfo fInfo = new FileInfo(sPath);
long numBytes = fInfo.Length;
//Open FileStream to read file
FileStream fStream = new FileStream(sPath, FileMode.Open, FileAccess.Read);
//Use BinaryReader to read file stream into byte array.
BinaryReader br = new BinaryReader(fStream);
//When you use BinaryReader, you need to supply number of bytes to read from file.
//In this case we want to read entire file. So supplying total number of bytes.
data = br.ReadBytes((int)numBytes);
return data;
}
void GetImagesFromDatabase()
{
try
{
//Initialize Oracle connection.
con = new OracleConnection(conString);
//MessageBox.Show("Connection Successfull");
//Initialize Oracle adapter.
OracleDataAdapter oda = new OracleDataAdapter("Select * from Imgpn", con);
//Initialize Dataset.
DataSet DS = new DataSet();
//Fill dataset with ImagesStore table.
oda.Fill(DS, "Imgpn");
//Fill Grid with dataset.
dataGridView1.DataSource = DS.Tables["Imgpn"];
//
}
catch (Exception ex)
{
MessageBox.Show(ex.ToString());
}
}
here is the simple way to insert image into oracle database ane retrieve ane show in datagridview