I am compressing the bytes and again while decompressing it I get OOM exception. I am not able to understand why am I getting this error when I have enough memory to store it.
The data is around 20MB after being compressed that is to be decompressed. But I always get OutOfMemory exception.
Below is the code for the same.
public byte[] Compress(byte[] data)
{
byte[] compressArray = null;
try
{
using (MemoryStream memoryStream = new MemoryStream())
{
using (DeflateStream deflateStream = new DeflateStream(memoryStream, CompressionMode.Compress))
{
deflateStream.Write(data, 0, data.Length);
deflateStream.Close();
}
compressArray = memoryStream.GetBuffer();
memoryStream.Dispose();
}
}
catch (Exception exception)
{
LogManager.LogEvent(EventLogEntryType.Error, exception.Message);
return data;
}
finally { GC.Collect(); }
return compressArray;
}
public static byte[] Decompress_Bytes(byte[] data)// Around 20MB data
{
byte[] decompressedArray = null;
try
{
using (MemoryStream decompressedStream = new MemoryStream())
{
using (MemoryStream compressStream = new MemoryStream(data))
{
using (DeflateStream deflateStream = new DeflateStream(compressStream, CompressionMode.Decompress))
{
deflateStream.CopyTo(decompressedStream);// Exception thrown at this line.
deflateStream.Close();
}
compressStream.Dispose();
}
decompressedArray = decompressedStream.GetBuffer();
decompressedStream.Dispose();
}
}
catch (Exception exception)
{
return data;
}
finally { GC.Collect(); }
return decompressedArray;
}
Below is the stack trace for better understanding.
at System.IO.MemoryStream.set_Capacity(Int32 value)
at System.IO.MemoryStream.EnsureCapacity(Int32 value)
at System.IO.MemoryStream.Write(Byte[] buffer, Int32 offset, Int32 count)
at System.IO.Stream.InternalCopyTo(Stream destination, Int32 bufferSize)
at System.IO.Stream.CopyTo(Stream destination)
at Symtrax.SQConsole.ConsoleConnectClass.Decompress_Bytes(Byte[] data) in c:\Developement\BI\branch_5.0\MapDesignerUNICODE\ConsoleConnector\SQConsole\ConsoleConnectClass.cs:line 3710
I found many relavant questions regarding this but none of them seem to solve my issue.
Since I have less reputation points I am unable to comment. Hence had to post question. Thanks in advance.
As already stated in the comments you're getting the internal buffer with GetBuffer that has different length characteristics then just calling ToArray.
I have added some dump statements in your code so LINQPad can reveal what is happening:
public byte[] Compress(byte[] data)
{
byte[] compressArray = null;
data.Length.Dump("initial array length");
try
{
using (MemoryStream memoryStream = new MemoryStream())
{
using (DeflateStream deflateStream = new DeflateStream(memoryStream, CompressionMode.Compress))
{
deflateStream.Write(data, 0, data.Length);
deflateStream.Close();
}
memoryStream.GetBuffer().Length.Dump("buffer compress len");
compressArray = memoryStream.ToArray();
compressArray.Length.Dump("compress array len");
// no need to call Dispose, using does that for you
//memoryStream.Dispose();
}
}
catch (Exception exception)
{
exception.Dump();
return data;
}
finally { GC.Collect(); }
return compressArray;
}
public static byte[] Decompress_Bytes(byte[] data)// Around 20MB data
{
byte[] decompressedArray = null;
try
{
using (MemoryStream decompressedStream = new MemoryStream())
{
using (MemoryStream compressStream = new MemoryStream(data))
{
using (DeflateStream deflateStream = new DeflateStream(compressStream, CompressionMode.Decompress))
{
deflateStream.CopyTo(decompressedStream);// Exception thrown at this line.
deflateStream.Close();
}
// no need, using does that
//compressStream.Dispose();
}
decompressedStream.GetBuffer().Length.Dump("buffer decompress len");
decompressedArray = decompressedStream.ToArray();
decompressedArray.Length.Dump("decompress array len");
// no need, using does that
decompressedStream.Dispose();
}
}
catch (Exception exception)
{
exception.Dump();
return data;
}
finally { GC.Collect(); }
return decompressedArray;
}
This is the output:
initial array length
248404
buffer compress len
262144
compress array len
189849
buffer decompress len
327680
decompress array len
248404
As you can see from these numbers you'll have very a different length count. You could possible get away with those extra bytes if the Deflate protocol would allow for byte streams that have extra bytes.
The use of GetBuffer instead of ToArray might seem beneficial but I expect the memory allocation and CPU ticks needed for copying of the final array be neglect able, specially if the memory stream is disposed anyway. That fact actually reduces the memory footprint a bit.
If you still insist on re-using the memory stream buffer make sure to also return and provide the actual length in the buffer:
public byte[] Compress(byte[] data, out int len)
{
byte[] compressArray = null;
data.Length.Dump("initial array length");
try
{
using (MemoryStream memoryStream = new MemoryStream())
{
// keep the stream open, we need the length!
using (DeflateStream deflateStream = new DeflateStream(
memoryStream,
CompressionMode.Compress,
true))
{
deflateStream.Write(data, 0, data.Length);
deflateStream.Close();
}
// output length
len = (int) memoryStream.Length;
compressArray = memoryStream.GetBuffer();
}
}
catch (Exception exception)
{
exception.Dump();
len =-1;
return data;
}
finally { GC.Collect(); }
return compressArray;
}
public static byte[] Decompress_Bytes(byte[] data, ref int len)// Around 20MB data
{
byte[] decompressedArray = null;
try
{
using (MemoryStream decompressedStream = new MemoryStream())
{
// use the overload that let us limit the memorystream buffer
using (MemoryStream compressStream = new MemoryStream(data,0, len))
{
// keep the stream open
using (DeflateStream deflateStream = new DeflateStream(
compressStream,
CompressionMode.Decompress,
true))
{
deflateStream.CopyTo(decompressedStream);// Exception thrown at this line.
deflateStream.Close();
}
}
// output length
decompressedArray = decompressedStream.GetBuffer();
len = (int) decompressedStream.Length;
}
}
catch (Exception exception)
{
exception.Dump();
return data;
}
finally { GC.Collect(); }
return decompressedArray;
}
If you use above code you'll have call it like this:
int len;
var cmp = Compress(Encoding.UTF8.GetBytes(sb.ToString()), out len);
var dec = Decompress_Bytes(cmp,ref len);
Notice to use the bytes in dec you need to only take the first len number of bytes into account. Practically this is done by using Array.Copy which defeats this solution and brings us back to the one which does call ToArray...
Related
i have two method for compress and decompress base64 image data in sql
server
my decompress method for read data in sql its work but compressing
return wrong data please help me ...
public static Byte[] BinaryCompress(Byte[] InputStream)
{
using (MemoryStream ms = new MemoryStream())
{
using (GZipStream x = new GZipStream(ms, CompressionMode.Compress, true))
{
byte[] inputBytes = (byte[])InputStream;
x.Write(inputBytes, 0, inputBytes.Length);
}
return ms.ToArray();
}
}
public static Byte[] BinaryDecompress(Byte[] InputBinary)
{
byte[] inputBytes = (byte[])InputBinary;
using (MemoryStream msin = new MemoryStream(inputBytes))
{
using (GZipStream x = new GZipStream(msin, CompressionMode.Decompress))
{
using (MemoryStream msout = new MemoryStream())
{
int num = x.ReadByte();
while (num != -1)
{
msout.WriteByte((byte)num);
num = x.ReadByte();
}
return msout.ToArray(); ;
}
}
}
}
Could you please provide more specific exception by using below code?
public static Byte[] BinaryCompress(Byte[] InputStream)
{
try
{
MemoryStream ms = new MemoryStream();
try
{
GZipStream x = new GZipStream(ms, CompressionMode.Compress, true)
byte[] inputBytes = (byte[])InputStream;
x.Write(inputBytes, 0, inputBytes.Length);
}
catch (Exception ex)
{
Console.Write(ex.StackTrace);
}
return ms.ToArray();
}
catch (Exception e)
{
Console.WriteLine(e.StackTrace);
throw;
}
}
I want to get a NBT file, decompress it, modify it, and then re-compress it. The only problem is re-compressing. Unlike just using GZipStream and calling it a day, Minecraft (and standalone NBT editors) can't read it.
public static void PatchNBT()
{
var data = File.ReadAllBytes(#"level.temp"); // Reads all bytes of the uncompressed NBT
var GetDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetEntryAssembly().Location), "level.dat");
Console.WriteLine(GetDirectory);
Console.ReadKey();
if (File.Exists("level.dat"))
{
File.Move("level.dat", "level.bak");
}
Task.WaitAll();
File.WriteAllBytes(GetDirectory, CompressGZip(data));
}
static byte[] CompressGZip(byte[] raw)
{
using (MemoryStream memory = new MemoryStream())
{
using (GZipStream gzip = new GZipStream(memory, CompressionLevel.Optimal))
{
gzip.Write(raw, 0, raw.Length);
}
return memory.ToArray();
}
}
i'm using the following function to compress(thanks to http://www.dotnetperls.com/):
public static void CompressStringToFile(string fileName, string value)
{
// A.
// Write string to temporary file.
string temp = Path.GetTempFileName();
File.WriteAllText(temp, value);
// B.
// Read file into byte array buffer.
byte[] b;
using (FileStream f = new FileStream(temp, FileMode.Open))
{
b = new byte[f.Length];
f.Read(b, 0, (int)f.Length);
}
// C.
// Use GZipStream to write compressed bytes to target file.
using (FileStream f2 = new FileStream(fileName, FileMode.Create))
using (GZipStream gz = new GZipStream(f2, CompressionMode.Compress, false))
{
gz.Write(b, 0, b.Length);
}
}
and for decompress:
static byte[] Decompress(byte[] gzip)
{
// Create a GZIP stream with decompression mode.
// ... Then create a buffer and write into while reading from the GZIP stream.
using (GZipStream stream = new GZipStream(new MemoryStream(gzip), CompressionMode.Decompress))
{
const int size = 4096;
byte[] buffer = new byte[size];
using (MemoryStream memory = new MemoryStream())
{
int count = 0;
do
{
count = stream.Read(buffer, 0, size);
if (count > 0)
{
memory.Write(buffer, 0, count);
}
}
while (count > 0);
return memory.ToArray();
}
}
}
so my goal is actually compress log files and than to decompress them in memory and compare the uncompressed file to the original file in order to check that the compression succeeded and i'm able to open the compressed file successfuly.
the problem is that the uncompressed file is most of the time bigger than the original file and my compare check is failing altough the compression probably succeeded.
any idea why ?
btw here how i compare the uncompressed file to the original file:
static bool FileEquals(byte[] file1, byte[] file2)
{
if (file1.Length == file2.Length)
{
for (int i = 0; i < file1.Length; i++)
{
if (file1[i] != file2[i])
{
return false;
}
}
return true;
}
return false;
}
Try this method to compress a file:
public static byte[] Compress(byte[] raw)
{
using (MemoryStream memory = new MemoryStream())
{
using (GZipStream gzip = new GZipStream(memory,
CompressionMode.Compress, true))
{
gzip.Write(raw, 0, raw.Length);
}
return memory.ToArray();
}
}
}
And this to decompress :
static byte[] Decompress(byte[] gzip)
{
// Create a GZIP stream with decompression mode.
// ... Then create a buffer and write into while reading from the GZIP stream.
using (GZipStream stream = new GZipStream(new MemoryStream(gzip), CompressionMode.Decompress))
{
const int size = 4096;
byte[] buffer = new byte[size];
using (MemoryStream memory = new MemoryStream())
{
int count = 0;
do
{
count = stream.Read(buffer, 0, size);
if (count > 0)
{
memory.Write(buffer, 0, count);
}
}
while (count > 0);
return memory.ToArray();
}
}
}
}
Tell me if it worked.
Goodluck.
Think you'd be better off with the simplest API call, try Stream.CopyTo(). I can't find the error in your code. If I was working on it, I'd probably make sure everything is getting flushed properly.. can't recall if GZipStream is going to flush its output to FileStream when the using block closes.. but then you are also saying that the final file is larger, not smaller.
Anyhow, best policy in my experience.. don't rewrite gotcha prone code when you don't need to. At least you tested it ;)
I'm using the library ICSharpCode.SharpZipLib to compress text.
After the compress finish I convert to base64 string and store in database. When I need to know what's stored, I get the value from database and decompress.
But, only in one client computer, when I compress a text and try to decompress the following error is throw:
BZip2 input stream crc error
I don't know whats the cause of error, because in my computer all compress and decompress are working fine.
I need your help to know what's the cause of the error and how to fix it. Sorry my bad english.
This is my code of compress and decompress:
public static byte[] Compress(byte[] bytesToCompress)
{
MemoryStream ms = new MemoryStream();
Stream s = new BZip2OutputStream(ms);
s.Write(bytesToCompress, 0, bytesToCompress.Length);
s.Close();
return ms.ToArray();
}
public static string Compress(string stringToCompress, Encoding encoding)
{
if (String.IsNullOrEmpty(stringToCompress))
return String.Empty;
byte[] compressedData = Compress(encoding.GetBytes(stringToCompress));
string strOut = Convert.ToBase64String(compressedData);
return strOut;
}
public static string DeCompress(string stringToDecompress)
{
Encoding encoding = Encoding.Unicode;
string outString;
try
{
byte[] inArr = Convert.FromBase64String(stringToDecompress.Trim());
outString = encoding.GetString(DeCompress(inArr));
}
catch (NullReferenceException nEx)
{
return nEx.Message;
}
return outString;
}
public static byte[] DeCompress(byte[] bytesToDecompress)
{
byte[] writeData = new byte[4096];
Stream s2 = new BZip2InputStream(new MemoryStream(bytesToDecompress));
MemoryStream outStream = new MemoryStream();
while (true)
{
int size = s2.Read(writeData, 0, writeData.Length);
if (size > 0)
{
outStream.Write(writeData, 0, size);
}
else
{
break;
}
}
s2.Close();
byte[] outArr = outStream.ToArray();
outStream.Close();
return outArr;
}
I have the following constructor method which opens a MemoryStream from a file path:
MemoryStream _ms;
public MyClass(string filePath)
{
byte[] docBytes = File.ReadAllBytes(filePath);
_ms = new MemoryStream();
_ms.Write(docBytes, 0, docBytes.Length);
}
I need to change this to accept a Stream instead of a file path. Whats the easiest/most efficient way to get a MemoryStream from the Stream object?
In .NET 4, you can use Stream.CopyTo to copy a stream, instead of the home-brew methods listed in the other answers.
MemoryStream _ms;
public MyClass(Stream sourceStream)
_ms = new MemoryStream();
sourceStream.CopyTo(_ms);
}
Use this:
var memoryStream = new MemoryStream();
stream.CopyTo(memoryStream);
This will convert Stream to MemoryStream.
If you're modifying your class to accept a Stream instead of a filename, don't bother converting to a MemoryStream. Let the underlying Stream handle the operations:
public class MyClass
{
Stream _s;
public MyClass(Stream s) { _s = s; }
}
But if you really need a MemoryStream for internal operations, you'll have to copy the data out of the source Stream into the MemoryStream:
public MyClass(Stream stream)
{
_ms = new MemoryStream();
CopyStream(stream, _ms);
}
// Merged From linked CopyStream below and Jon Skeet's ReadFully example
public static void CopyStream(Stream input, Stream output)
{
byte[] buffer = new byte[16*1024];
int read;
while((read = input.Read (buffer, 0, buffer.Length)) > 0)
{
output.Write (buffer, 0, read);
}
}
You can simply do:
var ms = new MemoryStream(File.ReadAllBytes(filePath));
Stream position is 0 and ready to use.
You will have to read in all the data from the Stream object into a byte[] buffer and then pass that into the MemoryStream via its constructor. It may be better to be more specific about the type of stream object you are using. Stream is very generic and may not implement the Length attribute, which is rather useful when reading in data.
Here's some code for you:
public MyClass(Stream inputStream) {
byte[] inputBuffer = new byte[inputStream.Length];
inputStream.Read(inputBuffer, 0, inputBuffer.Length);
_ms = new MemoryStream(inputBuffer);
}
If the Stream object doesn't implement the Length attribute, you will have to implement something like this:
public MyClass(Stream inputStream) {
MemoryStream outputStream = new MemoryStream();
byte[] inputBuffer = new byte[65535];
int readAmount;
while((readAmount = inputStream.Read(inputBuffer, 0, inputBuffer.Length)) > 0)
outputStream.Write(inputBuffer, 0, readAmount);
_ms = outputStream;
}
I use this combination of extension methods:
public static Stream Copy(this Stream source)
{
if (source == null)
return null;
long originalPosition = -1;
if (source.CanSeek)
originalPosition = source.Position;
MemoryStream ms = new MemoryStream();
try
{
Copy(source, ms);
if (originalPosition > -1)
ms.Seek(originalPosition, SeekOrigin.Begin);
else
ms.Seek(0, SeekOrigin.Begin);
return ms;
}
catch
{
ms.Dispose();
throw;
}
}
public static void Copy(this Stream source, Stream target)
{
if (source == null)
throw new ArgumentNullException("source");
if (target == null)
throw new ArgumentNullException("target");
long originalSourcePosition = -1;
int count = 0;
byte[] buffer = new byte[0x1000];
if (source.CanSeek)
{
originalSourcePosition = source.Position;
source.Seek(0, SeekOrigin.Begin);
}
while ((count = source.Read(buffer, 0, buffer.Length)) > 0)
target.Write(buffer, 0, count);
if (originalSourcePosition > -1)
{
source.Seek(originalSourcePosition, SeekOrigin.Begin);
}
}
How do I copy the contents of one stream to another?
see that. accept a stream and copy to memory. you should not use .Length for just Stream because it is not necessarily implemented in every concrete Stream.
public static void Do(Stream in)
{
_ms = new MemoryStream();
byte[] buffer = new byte[65536];
while ((int read = input.Read(buffer, 0, buffer.Length))>=0)
_ms.Write (buffer, 0, read);
}
byte[] fileData = null;
using (var binaryReader = new BinaryReader(Request.Files[0].InputStream))
{
fileData = binaryReader.ReadBytes(Request.Files[0].ContentLength);
}