We have a lot of files, saved as binary in our SQL Server database.
I have made an .ashx file, that delivers these files, to the users.
Unfortunately, when the files become rather large, it will fail, with the following error:
Overflow or underflow in the arithmetic operation
I assume it runs out of memory, as I load the binary into a byte[].
So, my question is, how can I make this functionality, read in chunks (maybe?), when it is from a database table? It also seems like Response.TransmitFile() is a good option, but again, how would this work with a database?
The DB.GetReposFile(), in the code beneath, gets the file from the database. There are various fields, for the entry:
Filename, ContentType, datestamps and the FileContent as varbinary.
This is my function, to deliver the file:
context.Response.Clear();
try
{
if (!String.IsNullOrEmpty(context.Request.QueryString["id"]))
{
int id = Int32.Parse(context.Request.QueryString["id"]);
DataTable dtbl = DB.GetReposFile(id);
string FileName = dtbl.Rows[0]["FileName"].ToString();
string Extension = FileName.Substring(FileName.LastIndexOf('.')).ToLower();
context.Response.ContentType = ReturnExtension(Extension);
context.Response.AddHeader("Content-Disposition", "attachment; filename=" + FileName);
byte[] buffer = (byte[])dtbl.Rows[0]["FileContent"];
context.Response.OutputStream.Write(buffer, 0, buffer.Length);
}
else
{
context.Response.ContentType = "text/html";
context.Response.Write("<p>Need a valid id</p>");
}
}
catch (Exception ex)
{
context.Response.ContentType = "text/html";
context.Response.Write("<p>" + ex.ToString() + "</p>");
}
Update:
The function I ended up with, is the one listed below.
DB.GetReposFileSize() simply gets the content Datalength, as Tim mentions.
I call this function, in the original code, instead of these two lines:
byte[] buffer = (byte[])dtbl.Rows[0]["FileContent"];
context.Response.OutputStream.Write(buffer, 0, buffer.Length);
New download function:
private void GetFileInChunks(HttpContext context, int ID)
{
//string path = #"c:\somefile.txt";
//FileInfo file = new FileInfo(path);
int len = DB.GetReposFileSize(ID);
context.Response.AppendHeader("content-length", len.ToString());
context.Response.Buffer = false;
//Stream outStream = (Stream)context.Response.OutputStream;
SqlConnection conn = null;
string strSQL = "select FileContent from LM_FileUploads where ID=#ID";
try
{
DB.OpenDB(ref conn, DB.DatabaseConnection.PDM);
SqlCommand cmd = new SqlCommand(strSQL, conn);
cmd.Parameters.AddWithValue("#ID", ID);
SqlDataReader reader = cmd.ExecuteReader(CommandBehavior.SequentialAccess);
reader.Read();
byte[] buffer = new byte[1024];
int bytes;
long offset = 0;
while ((bytes = (int)reader.GetBytes(0, offset, buffer, 0, buffer.Length)) > 0)
{
// TODO: do something with `bytes` bytes from `buffer`
context.Response.OutputStream.Write(buffer, 0, buffer.Length);
offset += bytes;
}
}
catch (Exception ex)
{
throw ex;
}
finally
{
DB.CloseDB(ref conn);
}
}
You can use DATALENGTH to get the size of the VARBINARY and stream it for instance with a SqldataReader and it's Read-or ReadBytes-Method.
Have a look at this answer to see an implementation: Best way to stream files in ASP.NET
Related
I am trying to send files with a bunch of strings from Android client to C# server. The strings, among other things, will also contain details with regards to the file being sent eg: File Size, File Name, etc. The issue I am having is that all the file bytes is not received thus the original file cannot be reconstructed at the destination even though I can properly retrieve all the strings.
My C# Server Code
while (true)
{
Socket socket = listener.AcceptSocket();
setStatus(socket.RemoteEndPoint + " Connected");
try
{
// Open the stream
NetworkStream stream = new NetworkStream(socket);
System.IO.StreamReader sr = new StreamReader(stream);
//Get string data
//********************************************
Teststr = sr.ReadLine();
setStatus(Teststr);
FileSize = sr.ReadLine();
long fileSizeLong = Convert.ToInt64(FileSize);
int length = (int)fileSizeLong;
setStatus("File size: " + length + " bytes");
//********************************************
//read bytes to buffer
byte[] buffer = new byte[length];
int toRead = (int)length;
int read = 0;
while (toRead > 0)
{
int noChars = stream.Read(buffer, read, toRead);
read += noChars;
toRead -= noChars;
}
setStatus("File Recieved. Total bytes: " + Convert.ToString(buffer.Length));
setStatus("Saving File");
String recievedPath = "C:\\Test\\";
BinaryWriter bWrite = new BinaryWriter(File.Open(recievedPath + "Test.png", FileMode.Create));
bWrite.Write(buffer);
setStatus("File Saved to: " + recievedPath);
bWrite.Flush();
bWrite.Close();
stream.Flush();
stream.Close();
}
catch (Exception e)
{
setStatus(e.Message);
}
setStatus("Disconnected");
socket.Close();
}
My Android Client Code
File file = new File(configstr[2]); //create file instance
try {
client = new Socket(configstr[0], Integer.valueOf(configstr[1]));
//Read file
fileInputStream = new FileInputStream(file);
outputStream = client.getOutputStream();
//Output database details to stream
//*****************************************
//Holds the string before conversion to bytes
String text = "";
text = "Test\n";
outputStream.write(text.getBytes());
text = String.valueOf(file.length()) + "\n";
outputStream.write(text.getBytes());
//*****************************************
outputStream.flush();
ByteArrayOutputStream bos = new ByteArrayOutputStream();
byte[] b = new byte[1024];
int bytesRead = 0;
while ((bytesRead = fileInputStream.read(b)) != -1) {
bos.write(b, 0, bytesRead);
}
byte[] bytes = bos.toByteArray();
outputStream.write(bytes);
outputStream.flush();
return true;
} catch (UnknownHostException e) {
e.printStackTrace();
return false;
} catch (IOException e) {
e.printStackTrace();
return false;
}
NOTE: The code works well if I only send one string, eg: If I only send file size.
Any better way of getting this to work as I am trying to send many strings through the stream as well as binary data from the file.
BufferedWriter with flush() at the end of every write() solved the issue for me.
My problem happens when trying to download a video file via http request
in the following operating systems using IIS7:
win2008 32Bit, Win2008 R2 64Bit
Currently works fine on: win2003 , vista64 (IIS6)
Problem description:
When users request a file larger than 256mb via C# they get a limited file, even when
using Content-Length param it seems that the file get the right size but not the full
content.
When requesting the URL of the file, I get the full file, the problem occurs only via
the C# script, also the C# response that the full buffer was sent to the user.
I've changed the IIS7 settings in the article:
http://blog.twinharbor.com/2011/07/28/fixing-iis7-maximum-upload-size/
and still it doesn't work.
Also, there are no remarks or errors anywhere.
Please find a sample of my code:
var context = System.Web.HttpContext.Current;
context.Response.ContentEncoding = Encoding.GetEncoding("windows-1255");
context.Response.HeaderEncoding = Encoding.GetEncoding("UTF-8");
context.Response.Charset = "utf-8";
System.IO.Stream iStream = null;
// Buffer to read 100K bytes in chunk:
byte[] buffer = new Byte[100000];
// Length of the file:
int length=0;
// Total bytes to read:
long dataToRead=0;
// Identify the file to download including its path.
string filepath = u.Trim(BigFile);
// Identify the file name.
string filename = System.IO.Path.GetFileName(filepath);
Start.Value = u.Time();
try
{
iStream = new System.IO.FileStream(filepath, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read);
dataToRead = iStream.Length;
context.Response.Charset = "";
context.Response.ContentType = "application/octet-stream";
context.Response.AddHeader("Content-Disposition", "attachment; filename=" + filename);
context.Response.AddHeader("Content-Length", dataToRead.ToString());
while (dataToRead > 0)
{
if (context.Response.IsClientConnected)
{
length = iStream.Read(buffer, 0, 100000);
context.Response.OutputStream.Write(buffer, 0, length);
context.Response.Flush();
buffer = new Byte[100000];
dataToRead = dataToRead - length;
}
else
{
dataToRead = -1;
}
}
}
catch (Exception ex)
{
context.Response.Write("Error : " + ex.Message);
}
finally
{
if (iStream != null)
{
iStream.Close();
}
context.Response.Close();
}
I'll appericiate your help.
Thanks.
I have this download function, and it's working great. BUT with a file with filesize of 1.35gb the download stops at 300 Mb, 382, 400mb or 1.27 Gb. What am I doing wrong? (The download function is made this way, because files need to be hidden, and may not be published on the website.)
public static void downloadFunction(string filename)
{
string filepath = #"D:\texts\New folder\DLfolder\" + filename;
string contentType = "application/x-newton-compatible-pkg";
Stream iStream = null;
// Buffer to read 10K bytes in chunk
//byte[] buffer = new Byte[10000];
// Buffer to read 1024K bytes in chunk
byte[] buffer = new Byte[1048576];
// Length of the file:
int length;
// Total bytes to read:
long dataToRead;
try
{
// Open the file.
iStream = new FileStream(filepath, FileMode.Open, FileAccess.Read, FileShare.Read);
// Total bytes to read:
dataToRead = iStream.Length;
HttpContext.Current.Response.ContentType = contentType;
HttpContext.Current.Response.AddHeader("Content-Disposition", "attachment; filename=" + HttpUtility.UrlEncode(filename, System.Text.Encoding.UTF8));
HttpContext.Current.Response.AddHeader("Content-Length", iStream.Length.ToString());
// Read the bytes.
while (dataToRead > 0)
{
// Verify that the client is connected.
if (HttpContext.Current.Response.IsClientConnected)
{
// Read the data in buffer.
length = iStream.Read(buffer, 0, 10000);
// Write the data to the current output stream.
HttpContext.Current.Response.OutputStream.Write(buffer, 0, length);
// Flush the data to the HTML output.
HttpContext.Current.Response.Flush();
buffer = new Byte[10000];
dataToRead = dataToRead - length;
}
else
{
// Prevent infinite loop if user disconnects
dataToRead = -1;
}
}
}
catch (Exception ex)
{
// Trap the error, if any.
HttpContext.Current.Response.Write("Error : " + ex.Message + "<br />");
HttpContext.Current.Response.ContentType = "text/html";
HttpContext.Current.Response.Write("Error : file not found");
}
finally
{
if (iStream != null)
{
//Close the file.
iStream.Close();
}
HttpContext.Current.Response.End();
HttpContext.Current.Response.Close();
}
}
You may have hit the request timeout. Take a note, if file stops downloading after some predetermined amount of time, like 60 or 300 seconds. If that is the case, you can configure timeouts in web.config of your application.
I am saving files to a SQL server 2008 (Express) database using FILESTREAM, the trouble I'm having is that certain files seem to be getting corrupted in the process.
For example if I save a word or excel document in one of the newer formats (docx, or xslx) then when I try to open the file I get an error message saying that the data is corrupted and would I like word/excel to try and recover it, If I click yes office is able to 'recover' the data and opens the file in compatibility mode.
However if i zip the file first then after extracting the contents I'm able to open the file without a problem. Strangely If I save an mp3 file to the database then I have the reverse issue, I can open the file no problem, but If I saved a zipped version of the mp3 I can't even extract the contents of that zip. When I tried to save a pdf or power-point file I ran into similar problems (the pdf i could only read if I zipped it first, and the ppt I couldn't read at all).
Update: here's my code that I'm using to write to the database and to read
To write to the database:
SQL = "SELECT Attachment.PathName(), GET_FILESTREAM_TRANSACTION_CONTEXT() FROM Activity " +
"WHERE RowID = CAST(#RowID as uniqueidentifier)";
transaction = connection.BeginTransaction();
command.Transaction = transaction;
command.CommandText = SQL;
command.Parameters.Clear();
command.Parameters.Add(rowIDParam);
SqlDataReader readerFS = null;
readerFS= command.ExecuteReader();
string path = (string)readerFS[0].ToString();
byte[] context = (byte[])readerFS[1];
int length = context.Length;
SqlFileStream targetStream = new SqlFileStream(path, context, FileAccess.Write);
int blockSize = 1024 * 512; //half a megabyte
byte[] buffer = new byte[blockSize];
int bytesRead = sourceStream.Read(buffer, 0, buffer.Length);
while (bytesRead > 0)
{
targetStream.Write(buffer, 0, bytesRead);
bytesRead = sourceStream.Read(buffer, 0, buffer.Length);
}
targetStream.Close();
sourceStream.Close();
readerFS.Close();
transaction.Commit();
And to read:
SqlConnection connection = null;
SqlTransaction transaction = null;
try
{
connection = getConnection();
connection.Open();
transaction = connection.BeginTransaction();
SQL = "SELECT Attachment.PathName(), + GET_FILESTREAM_TRANSACTION_CONTEXT() FROM Activity"
+ " WHERE ActivityID = #ActivityID";
SqlCommand command = new SqlCommand(SQL, connection);
command.Transaction = transaction;
command.Parameters.Add(new SqlParameter("ActivityID", activity.ActivityID));
SqlDataReader reader = command.ExecuteReader();
string path = (string)reader[0];
byte[] context = (byte[])reader[1];
int length = context.Length;
reader.Close();
SqlFileStream sourceStream = new SqlFileStream(path, context, FileAccess.Read);
int blockSize = 1024 * 512; //half a megabyte
byte[] buffer = new byte[blockSize];
List<byte> attachmentBytes = new List<byte>();
int bytesRead = sourceStream.Read(buffer, 0, buffer.Length);
while (bytesRead > 0)
{
bytesRead = sourceStream.Read(buffer, 0, buffer.Length);
foreach (byte b in buffer)
{
attachmentBytes.Add(b);
}
}
FileStream outputStream = File.Create(outputPath);
foreach (byte b in attachmentBytes)
{
byte[] barr = new byte[1];
barr[0] = b;
outputStream.Write(barr, 0, 1);
}
outputStream.Close();
sourceStream.Close();
command.Transaction.Commit();
Your read code is incorrect:
while (bytesRead > 0)
{
bytesRead = sourceStream.Read(buffer, 0, buffer.Length);
foreach (byte b in buffer)
{
attachmentBytes.Add(b);
}
}
If the bytesRead is less than buffer.Length, you still add the entire buffer to the attachementBytes. Thus, you always corrupt the document returned by adding any garbage in the end of the last buffer post bytesRead.
Other than that, allow me to have a really WTF moment. Reading a stream as a List<byte> ?? C'mon! First, I don't see the reason why you need to read into an intermediate in-memory storage to start with. You can simply read buffer by buffer and write each buffer straight into the outputStream. Second, if you must use an intermediate in-memory storage, use a MemoryStream, not a List<byte>.
I had the exact problem a few months back and figured out that I was adding an extra byte at the end of the file when reading it from FILESTREAM.
The code previously implemented takes in the xls file saves it on to a column in a table using the stream i use the same method but the only change is the the file saved is a xlsm or an xlsx type file it saves to the column in the database
When I try and get the contents from the database and throw the saved xlsm file or xlsx file I get an error "Excel file found unreadable content do you want to recover the contents of this work book ?"
Here's the code to save the xlsm or the xlsx file
System.IO.Stream filestream = System.IO.File.Open(file, System.IO.FileMode.Open);
int fileLength = (int)filestream.Length;
byte[] input = new byte[fileLength];
filestream.Read(input, 0, fileLength);
string Sql = "insert into upload values(#contents)";
con.Open();
System.Data.SqlClient.SqlCommand c = new System.Data.SqlClient.SqlCommand(Sql, con);
c.Parameters.Add("#contents", System.Data.SqlDbType.Binary);
c.Parameters["#contents"].Value = input;
c.ExecuteNonQuery();
To retrieve and send to user
SqlCommand comm = new SqlCommand("select contents from upload order by id desc", con);
SqlDataReader reader = comm.ExecuteReader();
int bufferSize = 32768;
byte[] outbyte = new byte[bufferSize];
long retval;
long startIndex = 0;
startIndex = 0;
retval = reader.GetBytes(0, startIndex, outbyte, 0, bufferSize);
while (retval > 0)
{
System.Web.HttpContext.Current.Response.BinaryWrite(outbyte);
startIndex += bufferSize;
if (retval == bufferSize)
{
retval = reader.GetBytes(2, startIndex, outbyte, 0, bufferSize);
}
else
{
retval = 0;
}
}
A couple of things strike me as possibilities.
Firstly, you are not calling reader.Read().
Secondly, there is not need for the check on retval == bufferSize - just call GetBytes again and it will return 0 if no bytes were read from the field.
Thirdly, as you are writing to the HttpResponse you need to make sure that you call Response.Clear() before writing the bytes to the output, and Response.End() after writing the file to the response.
The other thing to try is saving the file to the hard drive and comparing it to the original. Is it the same size? If it is bigger then you are writing too much information to the file (see previous comments about HttpResponse). If it is smaller then you are not writing enough, and are most likely exiting the loop too soon (see comment about retval).
I couldn't help but notice the number of places where your code failed to wrap an IDisposable in a using block, like the following:
using (SqlConnection con = new SqlConnection(connectionString))
{
byte[] input;
using (System.IO.Stream filestream = System.IO.File.Open(file, System.IO.FileMode.Open))
{
int fileLength = (int)filestream.Length;
input = new byte[fileLength];
filestream.Read(input, 0, fileLength);
}
const string Sql = "insert into upload values(#contents)";
con.Open();
using (System.Data.SqlClient.SqlCommand c = new System.Data.SqlClient.SqlCommand(Sql, con))
{
c.Parameters.Add("#contents", System.Data.SqlDbType.Binary);
c.Parameters["#contents"].Value = input;
c.ExecuteNonQuery();
}
using (SqlCommand comm = new SqlCommand("select contents from upload order by id desc", con))
{
using (SqlDataReader reader = comm.ExecuteReader())
{
int bufferSize = 32768;
byte[] outbyte = new byte[bufferSize];
long retval;
long startIndex = 0;
startIndex = 0;
retval = reader.GetBytes(0, startIndex, outbyte, 0, bufferSize);
while (retval > 0)
{
System.Web.HttpContext.Current.Response.BinaryWrite(outbyte);
startIndex += bufferSize;
if (retval == bufferSize)
{
retval = reader.GetBytes(2, startIndex, outbyte, 0, bufferSize);
}
else
{
retval = 0;
}
}
}
}
}