I am compressing a json using deflate compression technique and saving to sql server database. The json contains values from any culture ie. th-TH, zh-TW. The compressed string is getting saved successfully in database.
Json includes data like {"#id":"2113","description":"อาหารเช้าคอนติเนนทัล"}
Now when i read the same data from db, i convert it to bytes as
Encoding encoding = Encoding.UTF8;
encoding.GetBytes(data ?? string.Empty)
The compression like this
public static string Compress(this string data, CompressionTypeOptions compressionType)
{
var bytes = Compress(Encoding.UTF-8.GetBytes(data ?? string.Empty), compressionType);
return Encoding.UTF-8.GetString(bytes);
}
}
private static byte[] Compress(byte[] data, CompressionTypeOptions compressionType)
{
using (var memoryStream1 = new MemoryStream(data))
{
using (var memoryStream2 = new MemoryStream())
{
using (var compressionStream = CreateCompressionStream(compressionType, (Stream)memoryStream2,
CompressionMode.Compress))
{
CopyTo((Stream)memoryStream1, compressionStream);
compressionStream.Close();
return memoryStream2.ToArray();
}
}
}
}
Then decompressing like this
using (var memoryStream = new MemoryStream(data))
{
using (var compressionStream = CreateCompressionStream(compressionType, (Stream)memoryStream,
CompressionMode.Decompress))
return ReadAllBytesFromStream(compressionStream);
}
Here is ReadAllBytesFromStream definition
private static byte[] ReadAllBytesFromStream(Stream stream)
{
using (var memoryStream = new MemoryStream())
{
var buffer1 = new byte[1];
while (true)
{
int count = stream.Read(buffer1, 0, 1);
if (count != 0)
memoryStream.Write(buffer1, 0, count);
else
break;
}
var length = memoryStream.Length;
var buffer2 = new byte[length];
memoryStream.Position = 0L;
memoryStream.Read(buffer2, 0, (int)length);
return buffer2;
}
}
Getting error at int count = stream.Read(buffer1, 0, 1); as
'System.IO.InvalidDataException'
'Unknown block type. Stream might be corrupted.'
Any help is appreaciated
A friend of mine gave me the challenge to decompress an assembly that was packed with Fody.Costura. The assembly has a dll dependency that was embedded as a resource. I tried to extract this .zip resource with dotPeek and decompress it with this code here
public static void Decompress(string path)
{
using (var stream = File.OpenRead(path))
using (var compressStream = new DeflateStream(stream, CompressionMode.Decompress))
{
compressStream.Seek(0, SeekOrigin.Begin);
var fs = File.Create(path + ".decompressed");
compressStream.CopyTo(fs);
fs.Close();
}
}
This works when it comes to extracting the .zip but the result is quite unuseful
Is there a suitable solution to decompress this packed dll?
This is my simple C# Console App code (Framework 4), which I`m using simply by "drag and drop" (Costura Compressed) files over compiled (ConsoleApp1) executable.
using System;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Text.RegularExpressions;
namespace ConsoleApp1
{
class Program
{
static void CopyTo(Stream source, Stream destination) {
int count;
var array = new byte[81920];
while ((count = source.Read(array, 0, array.Length)) != 0) {
destination.Write(array, 0, count);
}
}
static Stream LoadStream(string fullname) {
FileStream stream = default(FileStream);
if (fullname.EndsWith(".zip")) {
using (stream = new FileStream(fullname, FileMode.Open)) {
using (var compressStream = new DeflateStream(stream, CompressionMode.Decompress)) {
var memStream = new MemoryStream();
CopyTo(compressStream, memStream);
memStream.Position = 0;
return memStream;
}
}
}
return stream;
}
static void Main(string[] args) {
Stream stream; Stream file;
string RefilePath = #"^.+[^\\]+\\"; string fullname; string newFile;
for (int i = 0; i < args.Count(); i++) {
fullname = args[i];
newFile = Regex.Replace(fullname, "\\.zip$", string.Empty);
Console.Write("{0} -> {1}\r\n",
Regex.Replace(fullname, RefilePath, string.Empty),
Regex.Replace(newFile, RefilePath, string.Empty));
try
{
stream = LoadStream(fullname);
using (file = File.Create(newFile)) {
CopyTo(stream, file);
}
}
catch (Exception ex) {
Console.Write("{0}", ex.ToString());
}
}
}
}
}
Based On Cameron MacFarland Answer
The code that Costura uses to decompress those resources is here.
https://github.com/Fody/Costura/blob/master/src/Costura.Template/Common.cs
static void CopyTo(Stream source, Stream destination)
{
var array = new byte[81920];
int count;
while ((count = source.Read(array, 0, array.Length)) != 0)
{
destination.Write(array, 0, count);
}
}
static Stream LoadStream(string fullname)
{
var executingAssembly = Assembly.GetExecutingAssembly();
if (fullname.EndsWith(".zip"))
{
using (var stream = executingAssembly.GetManifestResourceStream(fullname))
using (var compressStream = new DeflateStream(stream, CompressionMode.Decompress))
{
var memStream = new MemoryStream();
CopyTo(compressStream, memStream);
memStream.Position = 0;
return memStream;
}
}
return executingAssembly.GetManifestResourceStream(fullname);
}
To decompress those resources there is this project to.
I'm trying to create a ZIP archive with a simple demo text file using a MemoryStream as follows:
using (var memoryStream = new MemoryStream())
using (var archive = new ZipArchive(memoryStream , ZipArchiveMode.Create))
{
var demoFile = archive.CreateEntry("foo.txt");
using (var entryStream = demoFile.Open())
using (var streamWriter = new StreamWriter(entryStream))
{
streamWriter.Write("Bar!");
}
using (var fileStream = new FileStream(#"C:\Temp\test.zip", FileMode.Create))
{
stream.CopyTo(fileStream);
}
}
If I run this code, the archive file itself is created but foo.txt isn't.
However, if I replace the MemoryStream directly with the file stream, the archive is created correctly:
using (var fileStream = new FileStream(#"C:\Temp\test.zip", FileMode.Create))
using (var archive = new ZipArchive(fileStream, FileMode.Create))
{
// ...
}
Is it possible to use a MemoryStream to create the ZIP archive without the FileStream?
Thanks to ZipArchive creates invalid ZIP file, I got:
using (var memoryStream = new MemoryStream())
{
using (var archive = new ZipArchive(memoryStream, ZipArchiveMode.Create, true))
{
var demoFile = archive.CreateEntry("foo.txt");
using (var entryStream = demoFile.Open())
using (var streamWriter = new StreamWriter(entryStream))
{
streamWriter.Write("Bar!");
}
}
using (var fileStream = new FileStream(#"C:\Temp\test.zip", FileMode.Create))
{
memoryStream.Seek(0, SeekOrigin.Begin);
memoryStream.CopyTo(fileStream);
}
}
That indicated we need to call Dispose on ZipArchive before we can use it, which as Amir suggests is likely because it writes final bytes like checksum to the archive that makes it complete. But in order not close the stream so we can re-use it after you need to pass true as the third parameter to ZipArchive.
Just another version of zipping without writing any file.
string fileName = "export_" + DateTime.Now.ToString("yyyyMMddhhmmss") + ".xlsx";
byte[] fileBytes = here is your file in bytes
byte[] compressedBytes;
string fileNameZip = "Export_" + DateTime.Now.ToString("yyyyMMddhhmmss") + ".zip";
using (var outStream = new MemoryStream())
{
using (var archive = new ZipArchive(outStream, ZipArchiveMode.Create, true))
{
var fileInArchive = archive.CreateEntry(fileName, CompressionLevel.Optimal);
using (var entryStream = fileInArchive.Open())
using (var fileToCompressStream = new MemoryStream(fileBytes))
{
fileToCompressStream.CopyTo(entryStream);
}
}
compressedBytes = outStream.ToArray();
}
Set the position of the stream to the 0 before copying it to the zip stream.
using (var memoryStream = new MemoryStream())
{
using (var archive = new ZipArchive(memoryStream, ZipArchiveMode.Create, true))
{
var demoFile = archive.CreateEntry("foo.txt");
using (var entryStream = demoFile.Open())
using (var streamWriter = new StreamWriter(entryStream))
{
streamWriter.Write("Bar!");
}
}
using (var fileStream = new FileStream(#"C:\Temp\test.zip", FileMode.Create))
{
memoryStream.Position=0;
memoryStream.WriteTo(fileStream);
}
}
Working solution for MVC
public ActionResult Index()
{
string fileName = "test.pdf";
string fileName1 = "test.vsix";
string fileNameZip = "Export_" + DateTime.Now.ToString("yyyyMMddhhmmss") + ".zip";
byte[] fileBytes = System.IO.File.ReadAllBytes(#"C:\test\test.pdf");
byte[] fileBytes1 = System.IO.File.ReadAllBytes(#"C:\test\test.vsix");
byte[] compressedBytes;
using (var outStream = new MemoryStream())
{
using (var archive = new ZipArchive(outStream, ZipArchiveMode.Create, true))
{
var fileInArchive = archive.CreateEntry(fileName, CompressionLevel.Optimal);
using (var entryStream = fileInArchive.Open())
using (var fileToCompressStream = new MemoryStream(fileBytes))
{
fileToCompressStream.CopyTo(entryStream);
}
var fileInArchive1 = archive.CreateEntry(fileName1, CompressionLevel.Optimal);
using (var entryStream = fileInArchive1.Open())
using (var fileToCompressStream = new MemoryStream(fileBytes1))
{
fileToCompressStream.CopyTo(entryStream);
}
}
compressedBytes = outStream.ToArray();
}
return File(compressedBytes, "application/zip", fileNameZip);
}
You need to finish writing the memory stream then read the buffer back.
using (var memoryStream = new MemoryStream())
{
using (var archive = new ZipArchive(memoryStream, ZipArchiveMode.Create))
{
var demoFile = archive.CreateEntry("foo.txt");
using (var entryStream = demoFile.Open())
using (var streamWriter = new StreamWriter(entryStream))
{
streamWriter.Write("Bar!");
}
}
using (var fileStream = new FileStream(#"C:\Temp\test.zip", FileMode.Create))
{
var bytes = memoryStream.GetBuffer();
fileStream.Write(bytes,0,bytes.Length );
}
}
using System;
using System.IO;
using System.IO.Compression;
namespace ConsoleApplication
{
class Program`enter code here`
{
static void Main(string[] args)
{
using (FileStream zipToOpen = new FileStream(#"c:\users\exampleuser\release.zip", FileMode.Open))
{
using (ZipArchive archive = new ZipArchive(zipToOpen, ZipArchiveMode.Update))
{
ZipArchiveEntry readmeEntry = archive.CreateEntry("Readme.txt");
using (StreamWriter writer = new StreamWriter(readmeEntry.Open()))
{
writer.WriteLine("Information about this package.");
writer.WriteLine("========================");
}
}
}
}
}
}
Function to return stream that contain zip file
public static Stream ZipGenerator(List<string> files)
{
ZipArchiveEntry fileInArchive;
Stream entryStream;
int i = 0;
List<byte[]> byteArray = new List<byte[]>();
foreach (var file in files)
{
byteArray.Add(File.ReadAllBytes(file));
}
var outStream = new MemoryStream();
using (var archive = new ZipArchive(outStream, ZipArchiveMode.Create, true))
{
foreach (var file in files)
{
fileInArchive=(archive.CreateEntry(Path.GetFileName(file), CompressionLevel.Optimal));
using (entryStream = fileInArchive.Open())
{
using (var fileToCompressStream = new MemoryStream(byteArray[i]))
{
fileToCompressStream.CopyTo(entryStream);
}
i++;
}
}
}
outStream.Position = 0;
return outStream;
}
If you want , write zip to file stream.
using (var fileStream = new FileStream(#"D:\Tools\DBExtractor\DBExtractor\bin\Debug\test.zip", FileMode.Create))
{
outStream.Position = 0;
outStream.WriteTo(fileStream);
}
`
I'm late to the party, but there are scenarios where you can't access the ZipArchive's constructor to set the leaveOpen parameter and where you don't want the ZIP to be written to disk. In my case, the AsiceArchive class I'm using internally creates a ZipArchive but doesn't set leaveOpen to true.
I created a subclass of Stream that delegates all calls to an inner stream (a few clicks with ReSharper). This class is not disposable, so when the ZipArchive gets disposed, nothing happens to the inner stream.
public class NondisposingStreamWrapper : Stream
{
private readonly Stream _streamImplementation;
public NondisposingStreamWrapper(Stream inner) => _streamImplementation = inner;
public override void Flush() => _streamImplementation.Flush();
public override int Read(byte[] buffer, int offset, int count) => _streamImplementation.Read(buffer, offset, count);
public override long Seek(long offset, SeekOrigin origin) => _streamImplementation.Seek(offset, origin);
public override void SetLength(long value) => _streamImplementation.SetLength(value);
public override void Write(byte[] buffer, int offset, int count) => _streamImplementation.Write(buffer, offset, count);
public override bool CanRead => _streamImplementation.CanRead;
public override bool CanSeek => _streamImplementation.CanSeek;
public override bool CanWrite => _streamImplementation.CanWrite;
public override long Length => _streamImplementation.Length;
public override long Position
{
get => _streamImplementation.Position;
set => _streamImplementation.Position = value;
}
}
Use it like this:
using var memoryStream = new MemoryStream();
var output = new NondisposingStreamWrapper(memoryStream);
using (var archive = new ZipArchive(output, ZipArchiveMode.Create))
{
// add entries to archive
}
memoryStream.Flush();
memoryStream.Position = 0;
// write to file just for testing purposes
File.WriteAllBytes("out.zip", memoryStream.ToArray());
This is the way to convert a entity to XML File and then compress it:
private void downloadFile(EntityXML xml) {
string nameDownloadXml = "File_1.xml";
string nameDownloadZip = "File_1.zip";
var serializer = new XmlSerializer(typeof(EntityXML));
Response.Clear();
Response.ClearContent();
Response.ClearHeaders();
Response.AddHeader("content-disposition", "attachment;filename=" + nameDownloadZip);
using (var memoryStream = new MemoryStream())
{
using (var archive = new ZipArchive(memoryStream, ZipArchiveMode.Create, true))
{
var demoFile = archive.CreateEntry(nameDownloadXml);
using (var entryStream = demoFile.Open())
using (StreamWriter writer = new StreamWriter(entryStream, System.Text.Encoding.UTF8))
{
serializer.Serialize(writer, xml);
}
}
using (var fileStream = Response.OutputStream)
{
memoryStream.Seek(0, SeekOrigin.Begin);
memoryStream.CopyTo(fileStream);
}
}
Response.End();
}
Just in case, if anyone wants to save a dynamic zip file through SaveFileDialog.
var logFileName = "zip_filename.zip";
appLogSaver.FileName = logFileName;
appLogSaver.Filter = "LogFiles|*.zip";
appLogSaver.DefaultExt = "zip";
DialogResult resDialog = appLogSaver.ShowDialog();
if (resDialog.ToString() == "OK")
{
System.IO.FileStream fs = (System.IO.FileStream)appLogSaver.OpenFile();
using (var memoryStream = new MemoryStream())
{
using (var archive = new ZipArchive(memoryStream, ZipArchiveMode.Create, true))
{
var demoFile = archive.CreateEntry("foo.txt");
using (var entryStream = demoFile.Open())
{
using (var streamWriter = new StreamWriter(entryStream))
{
//read your existing file and put the content here
streamWriter.Write("Bar!");
}
}
var demoFile2 = archive.CreateEntry("foo2.txt");
using (var entryStream = demoFile2.Open())
{
using (var streamWriter = new StreamWriter(entryStream))
{
streamWriter.Write("Bar2!");
}
}
}
memoryStream.Seek(0, SeekOrigin.Begin);
memoryStream.CopyTo(fs);
}
fs.Close();
}
For me something like this was ok:
using (var memoryStream = new MemoryStream())
{
using (var archive = new ZipArchive(memoryStream, ZipArchiveMode.Create, true))
{
var file = archive.CreateEntry("file.json");
using var entryStream = file.Open();
using var streamWriter = new StreamWriter(entryStream);
streamWriter.WriteLine(someJsonLine);
}
}
private void button6_Click(object sender, EventArgs e)
{
//create With Input FileNames
AddFileToArchive_InputByte(new ZipItem[]{ new ZipItem( #"E:\b\1.jpg",#"images\1.jpg"),
new ZipItem(#"E:\b\2.txt",#"text\2.txt")}, #"C:\test.zip");
//create with input stream
AddFileToArchive_InputByte(new ZipItem[]{ new ZipItem(File.ReadAllBytes( #"E:\b\1.jpg"),#"images\1.jpg"),
new ZipItem(File.ReadAllBytes(#"E:\b\2.txt"),#"text\2.txt")}, #"C:\test.zip");
//Create Archive And Return StreamZipFile
MemoryStream GetStreamZipFile = AddFileToArchive(new ZipItem[]{ new ZipItem( #"E:\b\1.jpg",#"images\1.jpg"),
new ZipItem(#"E:\b\2.txt",#"text\2.txt")});
//Extract in memory
ZipItem[] ListitemsWithBytes = ExtractItems(#"C:\test.zip");
//Choese Files For Extract To memory
List<string> ListFileNameForExtract = new List<string>(new string[] { #"images\1.jpg", #"text\2.txt" });
ListitemsWithBytes = ExtractItems(#"C:\test.zip", ListFileNameForExtract);
// Choese Files For Extract To Directory
ExtractItems(#"C:\test.zip", ListFileNameForExtract, "c:\\extractFiles");
}
public struct ZipItem
{
string _FileNameSource;
string _PathinArchive;
byte[] _Bytes;
public ZipItem(string __FileNameSource, string __PathinArchive)
{
_Bytes=null ;
_FileNameSource = __FileNameSource;
_PathinArchive = __PathinArchive;
}
public ZipItem(byte[] __Bytes, string __PathinArchive)
{
_Bytes = __Bytes;
_FileNameSource = "";
_PathinArchive = __PathinArchive;
}
public string FileNameSource
{
set
{
FileNameSource = value;
}
get
{
return _FileNameSource;
}
}
public string PathinArchive
{
set
{
_PathinArchive = value;
}
get
{
return _PathinArchive;
}
}
public byte[] Bytes
{
set
{
_Bytes = value;
}
get
{
return _Bytes;
}
}
}
public void AddFileToArchive(ZipItem[] ZipItems, string SeveToFile)
{
MemoryStream memoryStream = new MemoryStream();
//Create Empty Archive
ZipArchive archive = new ZipArchive(memoryStream, ZipArchiveMode.Create, true);
foreach (ZipItem item in ZipItems)
{
//Create Path File in Archive
ZipArchiveEntry FileInArchive = archive.CreateEntry(item.PathinArchive);
//Open File in Archive For Write
var OpenFileInArchive = FileInArchive.Open();
//Read Stream
FileStream fsReader = new FileStream(item.FileNameSource, FileMode.Open, FileAccess.Read);
byte[] ReadAllbytes = new byte[4096];//Capcity buffer
int ReadByte = 0;
while (fsReader.Position != fsReader.Length)
{
//Read Bytes
ReadByte = fsReader.Read(ReadAllbytes, 0, ReadAllbytes.Length);
//Write Bytes
OpenFileInArchive.Write(ReadAllbytes, 0, ReadByte);
}
fsReader.Dispose();
OpenFileInArchive.Close();
}
archive.Dispose();
using (var fileStream = new FileStream(SeveToFile, FileMode.Create))
{
memoryStream.Seek(0, SeekOrigin.Begin);
memoryStream.CopyTo(fileStream);
}
}
public MemoryStream AddFileToArchive(ZipItem[] ZipItems)
{
MemoryStream memoryStream = new MemoryStream();
//Create Empty Archive
ZipArchive archive = new ZipArchive(memoryStream, ZipArchiveMode.Create, true);
foreach (ZipItem item in ZipItems)
{
//Create Path File in Archive
ZipArchiveEntry FileInArchive = archive.CreateEntry(item.PathinArchive);
//Open File in Archive For Write
var OpenFileInArchive = FileInArchive.Open();
//Read Stream
FileStream fsReader = new FileStream(item.FileNameSource, FileMode.Open, FileAccess.Read);
byte[] ReadAllbytes = new byte[4096];//Capcity buffer
int ReadByte = 0;
while (fsReader.Position != fsReader.Length)
{
//Read Bytes
ReadByte = fsReader.Read(ReadAllbytes, 0, ReadAllbytes.Length);
//Write Bytes
OpenFileInArchive.Write(ReadAllbytes, 0, ReadByte);
}
fsReader.Dispose();
OpenFileInArchive.Close();
}
archive.Dispose();
return memoryStream;
}
public void AddFileToArchive_InputByte(ZipItem[] ZipItems, string SeveToFile)
{
MemoryStream memoryStream = new MemoryStream();
//Create Empty Archive
ZipArchive archive = new ZipArchive(memoryStream, ZipArchiveMode.Create, true);
foreach (ZipItem item in ZipItems)
{
//Create Path File in Archive
ZipArchiveEntry FileInArchive = archive.CreateEntry(item.PathinArchive);
//Open File in Archive For Write
var OpenFileInArchive = FileInArchive.Open();
//Read Stream
// FileStream fsReader = new FileStream(item.FileNameSource, FileMode.Open, FileAccess.Read);
byte[] ReadAllbytes = new byte[4096];//Capcity buffer
int ReadByte = 4096 ;int TotalWrite=0;
while (TotalWrite != item.Bytes.Length)
{
if(TotalWrite+4096>item.Bytes.Length)
ReadByte=item.Bytes.Length-TotalWrite;
Array.Copy(item.Bytes, TotalWrite, ReadAllbytes, 0, ReadByte);
//Write Bytes
OpenFileInArchive.Write(ReadAllbytes, 0, ReadByte);
TotalWrite += ReadByte;
}
OpenFileInArchive.Close();
}
archive.Dispose();
using (var fileStream = new FileStream(SeveToFile, FileMode.Create))
{
memoryStream.Seek(0, SeekOrigin.Begin);
memoryStream.CopyTo(fileStream);
}
}
public MemoryStream AddFileToArchive_InputByte(ZipItem[] ZipItems)
{
MemoryStream memoryStream = new MemoryStream();
//Create Empty Archive
ZipArchive archive = new ZipArchive(memoryStream, ZipArchiveMode.Create, true);
foreach (ZipItem item in ZipItems)
{
//Create Path File in Archive
ZipArchiveEntry FileInArchive = archive.CreateEntry(item.PathinArchive);
//Open File in Archive For Write
var OpenFileInArchive = FileInArchive.Open();
//Read Stream
// FileStream fsReader = new FileStream(item.FileNameSource, FileMode.Open, FileAccess.Read);
byte[] ReadAllbytes = new byte[4096];//Capcity buffer
int ReadByte = 4096 ;int TotalWrite=0;
while (TotalWrite != item.Bytes.Length)
{
if(TotalWrite+4096>item.Bytes.Length)
ReadByte=item.Bytes.Length-TotalWrite;
Array.Copy(item.Bytes, TotalWrite, ReadAllbytes, 0, ReadByte);
//Write Bytes
OpenFileInArchive.Write(ReadAllbytes, 0, ReadByte);
TotalWrite += ReadByte;
}
OpenFileInArchive.Close();
}
archive.Dispose();
return memoryStream;
}
public void ExtractToDirectory(string sourceArchiveFileName, string destinationDirectoryName)
{
//Opens the zip file up to be read
using (ZipArchive archive = ZipFile.OpenRead(sourceArchiveFileName))
{
if (Directory.Exists(destinationDirectoryName)==false )
Directory.CreateDirectory(destinationDirectoryName);
//Loops through each file in the zip file
archive.ExtractToDirectory(destinationDirectoryName);
}
}
public void ExtractItems(string sourceArchiveFileName,List< string> _PathFilesinArchive, string destinationDirectoryName)
{
//Opens the zip file up to be read
using (ZipArchive archive = ZipFile.OpenRead(sourceArchiveFileName))
{
//Loops through each file in the zip file
foreach (ZipArchiveEntry file in archive.Entries)
{
int PosResult = _PathFilesinArchive.IndexOf(file.FullName);
if (PosResult != -1)
{
//Create Folder
if (Directory.Exists( destinationDirectoryName + "\\" +Path.GetDirectoryName( _PathFilesinArchive[PosResult])) == false)
Directory.CreateDirectory(destinationDirectoryName + "\\" + Path.GetDirectoryName(_PathFilesinArchive[PosResult]));
Stream OpenFileGetBytes = file.Open();
FileStream FileStreamOutput = new FileStream(destinationDirectoryName + "\\" + _PathFilesinArchive[PosResult], FileMode.Create);
byte[] ReadAllbytes = new byte[4096];//Capcity buffer
int ReadByte = 0; int TotalRead = 0;
while (TotalRead != file.Length)
{
//Read Bytes
ReadByte = OpenFileGetBytes.Read(ReadAllbytes, 0, ReadAllbytes.Length);
TotalRead += ReadByte;
//Write Bytes
FileStreamOutput.Write(ReadAllbytes, 0, ReadByte);
}
FileStreamOutput.Close();
OpenFileGetBytes.Close();
_PathFilesinArchive.RemoveAt(PosResult);
}
if (_PathFilesinArchive.Count == 0)
break;
}
}
}
public ZipItem[] ExtractItems(string sourceArchiveFileName)
{
List< ZipItem> ZipItemsReading = new List<ZipItem>();
//Opens the zip file up to be read
using (ZipArchive archive = ZipFile.OpenRead(sourceArchiveFileName))
{
//Loops through each file in the zip file
foreach (ZipArchiveEntry file in archive.Entries)
{
Stream OpenFileGetBytes = file.Open();
MemoryStream memstreams = new MemoryStream();
byte[] ReadAllbytes = new byte[4096];//Capcity buffer
int ReadByte = 0; int TotalRead = 0;
while (TotalRead != file.Length)
{
//Read Bytes
ReadByte = OpenFileGetBytes.Read(ReadAllbytes, 0, ReadAllbytes.Length);
TotalRead += ReadByte;
//Write Bytes
memstreams.Write(ReadAllbytes, 0, ReadByte);
}
memstreams.Position = 0;
OpenFileGetBytes.Close();
memstreams.Dispose();
ZipItemsReading.Add(new ZipItem(memstreams.ToArray(),file.FullName));
}
}
return ZipItemsReading.ToArray();
}
public ZipItem[] ExtractItems(string sourceArchiveFileName,List< string> _PathFilesinArchive)
{
List< ZipItem> ZipItemsReading = new List<ZipItem>();
//Opens the zip file up to be read
using (ZipArchive archive = ZipFile.OpenRead(sourceArchiveFileName))
{
//Loops through each file in the zip file
foreach (ZipArchiveEntry file in archive.Entries)
{
int PosResult = _PathFilesinArchive.IndexOf(file.FullName);
if (PosResult!= -1)
{
Stream OpenFileGetBytes = file.Open();
MemoryStream memstreams = new MemoryStream();
byte[] ReadAllbytes = new byte[4096];//Capcity buffer
int ReadByte = 0; int TotalRead = 0;
while (TotalRead != file.Length)
{
//Read Bytes
ReadByte = OpenFileGetBytes.Read(ReadAllbytes, 0, ReadAllbytes.Length);
TotalRead += ReadByte;
//Write Bytes
memstreams.Write(ReadAllbytes, 0, ReadByte);
}
//Create item
ZipItemsReading.Add(new ZipItem(memstreams.ToArray(),file.FullName));
OpenFileGetBytes.Close();
memstreams.Dispose();
_PathFilesinArchive.RemoveAt(PosResult);
}
if (_PathFilesinArchive.Count == 0)
break;
}
}
return ZipItemsReading.ToArray();
}
SQLiteConnection.Open does not throw exception when opening a file that is not a database.
private void openDatabase()
{
sqlite = new SQLiteConnection("Data Source=" + this.filePath + ";Version=3;");
try
{
sqlite.Open();
}
catch(SQLiteException e)
{
MessageBox.Show(e.Message + e.StackTrace);
}
}
How do I determine if a file is a SQLite Database?
Read the first 16 bytes and then check for the string "SQLite Format"
VB.Net
Dim bytes(16) As Byte
Using fs As New IO.FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)
fs.Read(bytes, 0, 16)
End Using
Dim chkStr As String = System.Text.ASCIIEncoding.ASCII.GetString(bytes)
Return chkStr.Contains("SQLite format")
Update 2
C#
byte[] bytes = new byte[17];
using (IO.FileStream fs = new IO.FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) {
fs.Read(bytes, 0, 16);
}
string chkStr = System.Text.ASCIIEncoding.ASCII.GetString(bytes);
return chkStr.Contains("SQLite format");
public static bool isSQLiteDatabase(string pathToFile)
{
bool result = false;
if (File.Exists(pathToFile)) {
using (FileStream stream = new FileStream(pathToFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
byte[] header = new byte[16];
for (int i = 0; i < 16; i++)
{
header[i] = (byte)stream.ReadByte();
}
result = System.Text.Encoding.UTF8.GetString(header).Contains("SQLite format 3");
stream.Close();
}
}
return result;
}
I handle with big files(of which capacities is minimum 500MB) to split and merge by c#.
I have to split the file into thousands of files, sort these files into some groups, and merge these by each group.
The minimum number of files are 10,000.
I implement the merge function by using the method Stream.CopyTo(). Here is the main part of that.
using (Stream writer = File.OpenWrite(outputFilePath))
{
int fileNum = filePaths.Count();
for (int i = 0; i < fileNum; i++)
{
using (Stream reader = File.OpenRead(filePaths.ElementAt(i)))
{ reader.CopyTo(writer); }
}
}
I've tested my program to split 500MB into 17000 files of 2 groups and to merge each group of 8500 files into one file.
The merging part takes about 80 seconds. I think it is pretty slow compared to splitting the same file which takes about 15~20 seconds
Is there any method which is faster than my code?
Your code looks fine but ElementAt is a code smell. Convert that to an array and use [i] instead. If you have 10K elements I'm positive you're wasting a lot of time.
Why not just use the Stream.CopyTo() method?
private static void CombineMultipleFilesIntoSingleFile(string inputDirectoryPath, string inputFileNamePattern, string outputFilePath)
{
string[] inputFilePaths = Directory.GetFiles(inputDirectoryPath, inputFileNamePattern);
Console.WriteLine("Number of files: {0}.", inputFilePaths.Length);
using (var outputStream = File.Create(outputFilePath))
{
foreach (var inputFilePath in inputFilePaths)
{
using (var inputStream = File.OpenRead(inputFilePath))
{
// Buffer size can be passed as the second argument.
inputStream.CopyTo(outputStream);
}
Console.WriteLine("The file {0} has been processed.", inputFilePath);
}
}
}
OR
Do it in chunks:
const int chunkSize = 2 * 1024; // 2KB
var inputFiles = new[] ;
using (var output = File.Create("output.dat"))
{
foreach (var file in inputFiles)
{
using (var input = File.OpenRead(file))
{
var buffer = new byte[chunkSize];
int bytesRead;
while ((bytesRead = input.Read(buffer, 0, buffer.Length)) > 0)
{
output.Write(buffer, 0, bytesRead);
}
}
}
}
Maybe try compressing the files?
using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
using System.IO.Compression;
class Program {
static void SaveCompressedFile(string filename, string data) {
FileStream fileStream = new FileStream(filename, FileMode.Create, FileAccess.Write);
GZipStream compressionStream = new GZipStream(fileStream, CompressionMode.Compress);
StreamWriter writer = new StreamWriter(compressionStream);
writer.Write(data);
writer.Close();
}
static string LoadCompressedFile(string filename) {
FileStream fileStream = new FileStream(filename, FileMode.Open, FileAccess.Read);
GZipStream compressionStream = new GZipStream(fileStream, CompressionMode.Decompress);
StreamReader reader = new StreamReader(compressionStream);
string data = reader.ReadToEnd();
reader.Close();
return data;
}
static void Main(string[] args) {
try {
string filename = "compressedFile.txt";
string sourceString = "Source String";
SaveCompressedFile(filename, sourceString);
FileInfo compressedFileData = new FileInfo(filename);
string recoveredString = LoadCompressedFile(filename);
} catch (IOException ex) {
Console.WriteLine(ex.ToString());
}
}
}
Source
Also check out the example of compressing a directory.
using System;
using System.Text;
using System.IO;
using System.IO.Compression;
namespace CmprDir
{
class Program
{
delegate void ProgressDelegate(string sMessage);
static void CompressFile(string sDir, string sRelativePath, GZipStream zipStream)
{
//Compress file name
char[] chars = sRelativePath.ToCharArray();
zipStream.Write(BitConverter.GetBytes(chars.Length), 0, sizeof(int));
foreach (char c in chars)
zipStream.Write(BitConverter.GetBytes(c), 0, sizeof(char));
//Compress file content
byte[] bytes = File.ReadAllBytes(Path.Combine(sDir, sRelativePath));
zipStream.Write(BitConverter.GetBytes(bytes.Length), 0, sizeof(int));
zipStream.Write(bytes, 0, bytes.Length);
}
static bool DecompressFile(string sDir, GZipStream zipStream, ProgressDelegate progress)
{
//Decompress file name
byte[] bytes = new byte[sizeof(int)];
int Readed = zipStream.Read(bytes, 0, sizeof(int));
if (Readed < sizeof(int))
return false;
int iNameLen = BitConverter.ToInt32(bytes, 0);
bytes = new byte[sizeof(char)];
StringBuilder sb = new StringBuilder();
for (int i = 0; i < iNameLen; i++)
{
zipStream.Read(bytes, 0, sizeof(char));
char c = BitConverter.ToChar(bytes, 0);
sb.Append(c);
}
string sFileName = sb.ToString();
if (progress != null)
progress(sFileName);
//Decompress file content
bytes = new byte[sizeof(int)];
zipStream.Read(bytes, 0, sizeof(int));
int iFileLen = BitConverter.ToInt32(bytes, 0);
bytes = new byte[iFileLen];
zipStream.Read(bytes, 0, bytes.Length);
string sFilePath = Path.Combine(sDir, sFileName);
string sFinalDir = Path.GetDirectoryName(sFilePath);
if (!Directory.Exists(sFinalDir))
Directory.CreateDirectory(sFinalDir);
using (FileStream outFile = new FileStream(sFilePath, FileMode.Create, FileAccess.Write, FileShare.None))
outFile.Write(bytes, 0, iFileLen);
return true;
}
static void CompressDirectory(string sInDir, string sOutFile, ProgressDelegate progress)
{
string[] sFiles = Directory.GetFiles(sInDir, "*.*", SearchOption.AllDirectories);
int iDirLen = sInDir[sInDir.Length - 1] == Path.DirectorySeparatorChar ? sInDir.Length : sInDir.Length + 1;
using (FileStream outFile = new FileStream(sOutFile, FileMode.Create, FileAccess.Write, FileShare.None))
using (GZipStream str = new GZipStream(outFile, CompressionMode.Compress))
foreach (string sFilePath in sFiles)
{
string sRelativePath = sFilePath.Substring(iDirLen);
if (progress != null)
progress(sRelativePath);
CompressFile(sInDir, sRelativePath, str);
}
}
static void DecompressToDirectory(string sCompressedFile, string sDir, ProgressDelegate progress)
{
using (FileStream inFile = new FileStream(sCompressedFile, FileMode.Open, FileAccess.Read, FileShare.None))
using (GZipStream zipStream = new GZipStream(inFile, CompressionMode.Decompress, true))
while (DecompressFile(sDir, zipStream, progress));
}
public static int Main(string[] argv)
{
if (argv.Length != 2)
{
Console.WriteLine("Usage: CmprDir.exe <in_dir compressed_file> | <compressed_file out_dir>");
return 1;
}
string sDir;
string sCompressedFile;
bool bCompress = false;
try
{
if (Directory.Exists(argv[0]))
{
sDir = argv[0];
sCompressedFile = argv[1];
bCompress = true;
}
else
if (File.Exists(argv[0]))
{
sCompressedFile = argv[0];
sDir = argv[1];
bCompress = false;
}
else
{
Console.Error.WriteLine("Wrong arguments");
return 1;
}
if (bCompress)
CompressDirectory(sDir, sCompressedFile, (fileName) => { Console.WriteLine("Compressing {0}...", fileName); });
else
DecompressToDirectory(sCompressedFile, sDir, (fileName) => { Console.WriteLine("Decompressing {0}...", fileName); });
return 0;
}
catch (Exception ex)
{
Console.Error.WriteLine(ex.Message);
return 1;
}
}
}
}
Source