My method takes a List and creates a ZIP file and a log file. It works with smaller amounts of files. But when I go for a large amount of files (tested with 11,000) it throws a SystemOutOfMemory exception.
By research I learned that my method probably puts a lot of load in the memory. So I put in the part were I flush the streamwriter and the zip archive. I probably have to do something to the file stream.
What is an efficient approach to solve this problem?
Here is the code:
public static void BackupFilesToZip(string directory, string fileFilter, string zipFilePath, bool backupInSubDir, string logFilePath, List<FileInfo> filesToBackup)
{
FileInfo logFile = new FileInfo(logFilePath);
FileInfo zipFile = new FileInfo(zipFilePath);
int numberOfFiles = filesToBackup.Count;
if (!Directory.Exists(zipFile.DirectoryName)) Directory.CreateDirectory(zipFile.DirectoryName);
using (FileStream zipToOpen = new FileStream(zipFile.FullName, FileMode.OpenOrCreate))
{
using (ZipArchive archive = new ZipArchive(zipToOpen, ZipArchiveMode.Update))
{
ZipArchiveEntry readmeEntry = archive.CreateEntry(logFile.Name);
using (StreamWriter writer = new StreamWriter(readmeEntry.Open()))
{
writer.WriteLine("This ZIP archive was created: " + DateTime.Now.ToString("MM/dd/yy HH:mm:ss"));
writer.WriteLine("ZIP File: " + zipFilePath);
writer.WriteLine("Source Directory: " + directory);
string backupInSubText = "yes";
if (!backupInSubDir) backupInSubText = "no";
writer.WriteLine("Subdirectories included: " + backupInSubText);
writer.WriteLine("Filter Critera: " + fileFilter);
writer.WriteLine("Number of Files selected: " + numberOfFiles + " (for # of files archived/skipped scroll down)");
writer.WriteLine("");
writer.WriteLine("File Log:");
int filesArchivedCounter = 0;
int filesSkippedCounter = 0;
int filesSum = 0;
TaskbarManager.Instance.SetProgressState(TaskbarProgressBarState.Normal);
foreach (FileInfo file in filesToBackup)
{
//ZipArchiveEntry readmeEntry = archive.CreateEntry(logFile.Name);
string DateTimeStampBegin = DateTime.Now.ToString("MM/dd/yy HH:mm:ss");
try
{
string relativePath = MakeRelativePath(directory, file.FullName);
archive.CreateEntryFromFile(file.FullName, relativePath);
writer.WriteLine(DateTimeStampBegin + " - " + DateTime.Now.ToString("HH:mm:ss") + " archived: " + file.FullName);
filesArchivedCounter++;
}
catch
{
writer.WriteLine(DateTimeStampBegin + " - " + " SKIPPED: " + file.FullName);
filesSkippedCounter++;
}
filesSum = filesSkippedCounter + filesArchivedCounter;
TaskbarManager.Instance.SetProgressValue(filesSum, numberOfFiles);
//write from memory to files every 75 items (to avoid out of memory exception)
if (filesSum % 75 == 0)
{
writer.Flush();
zipToOpen.Flush();
}
}
writer.WriteLine("");
writer.WriteLine("# of Files archived: " + filesArchivedCounter);
writer.WriteLine("# of Files skipped: " + filesSkippedCounter);
}
if (!String.IsNullOrEmpty(logFile.Name))
{
if (!Directory.Exists(logFile.DirectoryName)) Directory.CreateDirectory(logFile.DirectoryName);
readmeEntry.ExtractToFile(logFile.FullName, true);
}
TaskbarManager.Instance.SetProgressState(TaskbarProgressBarState.NoProgress);
}
}
}
All the string parameters of this method are just for the log file.
The problem in your implementation is that you are putting the log entry at the beginning of the archive and update it after adding new files, so zip cannot be flushed. You should write log to file, not the archive, and add it to the archive when all files are added.
Related
I am trying to increment the filename (e.g., "file1.csv", "file2.csv", etc.), each time a new file is generated. I followed this thread Increment the file name if the file already exists in c# but the solution is not useful for my case. What I want to do is check if the file exists in the first place and if it does write in it. If it doesn't create one and write. The problem is that if the file exists but it's from another user, I want the system to increment the file number and not write to the same file just because it exists. What I have so far:
public void saveFile()
{
int count = 0;
string title = "TimeStamp,Name,Trial,Time_spent-dist,Time_spent_tar\n";
string output = System.DateTime.Now.ToString("mm_ss_ffff") + "," +
currentScene.name.ToString() + "," +
trialNum.ToString() + "," +
timerDistractor.ToString() + "," +
timerTarget.ToString();
string fname = "User_" + count + ".csv";
string path = Path.Combine(Application.persistentDataPath, fname);
if (File.Exists(path))
{
File.AppendAllText(path, "\n" + output);
}
else
{
StreamWriter writer = new StreamWriter(path);
writer.WriteLine(title + "\n" + output);
writer.Close();
}
}
Any pointers?
I am trying to create a console app on network version 5.0 using visual studio. The purpose is to read all the PNG files in a directory, and make a JSON file with the code:
{
"format_version": "1.16.100",
"minecraft:texture_set": {
"color": "*Filename*",
"metalness_emissive_roughness": "*Filename_mer*"
}
}
In it. And yes this is for Minecraft. The filename and the filename_mer are automatically filled in by code, but that isn't my issue.
static void Main(string[] args)
{
Console.WriteLine("Goto " + Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData) + "\\Packages\\Microsoft.MinecraftUWP_8wekyb3d8bbwe\\LocalState\\games\\com.mojang\\resource_packs" + " And find the resource pack you want to generate files into");
string pathname = Console.ReadLine();
#region Message
Console.WriteLine();
Console.WriteLine("Looking for folder...");
#endregion
string path = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData) + "\\Packages\\Microsoft.MinecraftUWP_8wekyb3d8bbwe\\LocalState\\games\\com.mojang\\resource_packs\\" + pathname + "\\textures";
#region Message
Console.WriteLine();
Console.WriteLine("Folder found in");
Console.WriteLine(path);
#endregion
string[] directories = Directory.GetDirectories(path);
foreach (string paths in directories)
{
string[] files = Directory.GetFiles(paths);
foreach (string file in files)
{
string filenameWithType = file.Substring(file.LastIndexOf("\\") + 1);
string filename = filenameWithType.Substring(0, filenameWithType.LastIndexOf("."));
string thisPath = file.Substring(0, file.LastIndexOf("\\"));
if (filenameWithType.Substring(filenameWithType.LastIndexOf(".") + 1) == "png")
{
string newPath = thisPath + "\\" + filename + ".texture_set.json";
File.Create(newPath);
List<string> codeInFile = new List<string>();
codeInFile.Clear();
codeInFile.Add("{");
codeInFile.Add("\"format_version\": \"1.16.100\",");
codeInFile.Add("\"minecraft:texture_set\": {");
codeInFile.Add("\"color\": \"" + filename + "\",");
codeInFile.Add("\"metalness_emissive_roughness\": \"" + filename + "_mer\"");
codeInFile.Add("}");
codeInFile.Add("}");
TextWriter tw = new StreamWriter(newPath, false);
foreach (string line in codeInFile)
{
tw.WriteLine(line);
Console.WriteLine(line);
}
tw.Close();
string newPathtxt = thisPath + "\\" + filename + "_mer.png";
Bitmap bitmap = new Bitmap(file);
using (Bitmap b = new Bitmap(bitmap.Width, bitmap.Height))
{
using (Graphics g = Graphics.FromImage(b))
{
g.Clear(Color.Green);
}
b.Save(newPathtxt, ImageFormat.Png);
}
}
}
}
#region Message
Console.WriteLine("");
Console.WriteLine("All done, Your good to go!");
#endregion
Console.Read();
}
This is all my code, on the file. It reads a directory which you enter, looks through the textures file, and for each PNG file in there it creates a JSON file with the code specified earlier. Although when run the code gives me this error.
System.IO.IOException: 'The process cannot access the file 'C:\Users\https\AppData\Local\Packages\Microsoft.MinecraftUWP_8wekyb3d8bbwe\LocalState\games\com.mojang\resource_packs\Vanilla_Resource_Pack_1.17.10\textures\blocks\acacia_trapdoor.texture_set.json' because it is being used by another process.'
I cannot find any answers which fit my issue, I would appreciate any help.
This is a Minecraft Bedrock edition resource pack btw, not sure if that helps.
Don't do this File.Create(newPath); or rethink your problem. It looks like a typo.
In short File.Create(newPath) is creating a FileStream and discarding it, leaving the file open and with a share lock. If you are trying to pre-create the file, at least use the using statement:
using (File.Create(newPath));
or Close
File.Create(newPath).Close();
or
File.WriteAllText(newPath, String.Empty);
or
File.WriteAllBytes(newPath, Array.Empty<byte>());
If you are just trying to create or overwrite the file StreamWriter(newPath, false) is good enough.
I'm doing a c# wcf service in which I receive a bunch of images and the service merge them in a multiimage Tiff file. At the end of the service I want to delete the original files but I'm receiving an error that some other process is locking the file.
This is the code that receives the images (as a byte[] list) and write them to disk
public static List<string> SaveByteImagesToFile(List<byte[]> bytesToCopyIntoFiles, string imageReferenceType, string imageReferenceValue)
{
_applicationLogger.Debug(MethodBase.GetCurrentMethod().DeclaringType.Name, MethodBase.GetCurrentMethod().Name);
string imageFinalPath = string.Empty;
string joinImagesFilePath = string.Empty;
List<string> imagesFilePath = new List<string>();
int count = 1;
try
{
if (bytesToCopyIntoFiles.Count == 0)
{
throw new ArgumentNullException("bytesToCopyIntoFiles");
}
else
{
joinImagesFilePath = SettingsManager.GetServiceSetting(AppSettingsKeys.CopyImagesToFilePath, "NO_VALID_FILEPATH");
if (joinImagesFilePath.IsValidFilePath(out string errorMessage, true, true))
{
foreach (byte[] image in bytesToCopyIntoFiles)
{
var imageFileName = imageReferenceType + "_" + imageReferenceValue + "_" + DateTime.Now.ToString("yyyyMMddHHmmssfff") + count.ToString();
imageFinalPath = joinImagesFilePath + Path.DirectorySeparatorChar + imageFileName + ".tiff";
using (FileStream stream = new FileStream(imageFinalPath, FileMode.Create, FileAccess.ReadWrite))
{
stream.Write(image, 0, image.Length);
stream.Flush();
}
imagesFilePath.Add(imageFinalPath);
count++;
}
}
else
{
exceptionMessageType = MainRepository.GetExceptionMessage("E171");
throw new IOException(exceptionMessageType.ExceptionMessage + " " + errorMessage);
}
}
return imagesFilePath;
}
catch
{
throw;
}
}
How or what can I use to prevent the service or any process to lock the file. As you can see I'm using the using scope for filestream without any luck.
Any ideas? Thanks
Resolved! By organizing the files in a certain order, when creating the multipage tiff, by the time the logic ends the worker already unlock the resources and I'm able now to delete them without any issue.
I am creating application to delete files for more than 15 days in past, I've created a project using the C# language "multithreading" to be able to delete these files, but its only reading the first file with the error
The directory name is invalid
Can anyone help me on this please?
private void process3()
{
//DirectoryInfo info1 = new DirectoryInfo(#"\\10.4.9.202\d\PapyrusRes\appdata\");
DirectoryInfo info1 = new DirectoryInfo(#"\\DXB-RASO-MCH\Users\oalahmad\Dropbox\backup\Backup5\Desktop\New folder2");
// long Size = 0;
//C:\Users\oalahmad\Dropbox\backup\Backup5\Desktop\New folder2
String[] filePaths = (from fls in info1.EnumerateFiles()
where (fls.LastWriteTime.Date < DateTime.Today.AddDays(-15))
select fls.FullName).ToArray();
int i = 0;
if (!File.Exists(logPath3))
{
// Create a file to write to.
using (StreamWriter sw = File.CreateText(logPath3))
{
sw.WriteLine("Deletion Process History:");
sw.WriteLine(" ");
sw.WriteLine(" ");
}
}
//stopwatch.Start();
try
{
foreach (String f in filePaths)
{
DirectoryInfo info = new DirectoryInfo(f);
int difference = DateTime.Today.Subtract(info.LastWriteTime).Days;
textBox2.BeginInvoke(new Action(() =>
{
textBox2.Text += "Folder Name: " + Path.GetFileName(f) +
"\r\nDate Modified: " + difference +
"\r\n------\r\n";
}));
Thread.Sleep(10);
i++;
Directory.Delete(f, true);
count++;
}
using (StreamWriter sw = File.AppendText(logPath3))
{
sw.WriteLine("Successful at: " + DateTime.Now + " " + count +
" files were deleted");
}
}
catch (Exception ex)
{
// log errors
// Write your content here
using (StreamWriter sw = File.AppendText(logPath3))
{
if (count == 0)
sw.WriteLine("Unsuccessful at: " + DateTime.Now + " Error: " +
ex.Message);
else
sw.WriteLine("Unsuccessful at: " + DateTime.Now + " " + count +
" files were deleted" + " Error: " + ex.Message);
}
}
}
I wanted to write a text to file using StreamWriter.But Filename should be current date name.
here is my coding.Can somebody tell me how to specify the file creation path?
Code Edit :
In here i wanted to create a .txt file but in here file not created.
public void WriteToFile( string name, string source, int dest, string messageIn, string operatorNew)
{
string directory = ResolveUrl("~/DesktopModules/SMSFunction/SMSText");
string filename = String.Format("{0:yyyy-MM-dd}__{1}", DateTime.Now,name);
string path = Path.Combine(directory, filename);
if (!File.Exists(filename))
{
using (StreamWriter str = File.CreateText(path))
{
str.WriteLine("msisdn: " + source);
str.WriteLine("shortcode : " + dest);
str.WriteLine("Message : " + messageIn);
str.WriteLine("Operator :" + operatorNew);
str.Flush();
}
}
else if (File.Exists(filename))
{
using (var str = new StreamWriter(filename))
{
str.WriteLine("msisdn: " + source);
str.WriteLine("shortcode : " + dest);
str.WriteLine("Message : " + messageIn);
str.WriteLine("Operator :" + operatorNew);
str.Flush();
}
}
you need to make following changes
1.Replace ResolveUrl with Server.MapPath
string directory = Server.MapPath("~/DesktopModules/SMSFunction/SMSText");
2.Add the file extension .txt as shown below
string filename = String.Format("{0:yyyy-MM-dd}__{1}.txt", DateTime.Now,name);
3.when you are checking whether file exists or not provide the path of the file , instead of filename
File.Exists(path);
4.under the else if block , here also provide the path , instead of filename
var str = new StreamWriter(path));
putting all together the code looks like,
string directory = Server.MapPath("~/DesktopModules/SMSFunction/SMSText");
string filename = String.Format("{0:yyyy-MM-dd}__{1}.txt", DateTime.Now, name);
string path = Path.Combine(directory, filename);
if (!File.Exists(path))
{
using (StreamWriter str = File.CreateText(path))
{
str.WriteLine("msisdn: " + source);
str.WriteLine("shortcode : " + dest);
str.WriteLine("Message : " + messageIn);
str.WriteLine("Operator :" + operatorNew);
str.Flush();
}
}
else if (File.Exists(path))
{
using (var str = new StreamWriter(path))
{
str.WriteLine("msisdn: " + source);
str.WriteLine("shortcode : " + dest);
str.WriteLine("Message : " + messageIn);
str.WriteLine("Operator :" + operatorNew);
str.Flush();
}
File.Create returns FileStream, and you need StreamWriter. You'll have to use its constructor that accepts Stream:
using (var str = new StreamWriter(File.CreateText(path)))
Simplify, use FileStream to create or overwrite your file (see below) depending on your neeeds you might want to change the FileMode to be something else (Append ?)
public void WriteToFile(string name, string source, int dest, string messageIn, string operatorNew)
{
string directory = ResolveUrl("~/DesktopModules/SMSFunction/SMSText");
string filename = String.Format("{0:yyyy-MM-dd}__{1}", DateTime.Now,name);
string path = Path.Combine(directory, filename);
using (FileStream fs = new FileStream(path, FileMode.Create))
{
using (StreamWriter str = new StreamWriter(fs))
{
str.WriteLine("msisdn: " + source);
str.WriteLine("shortcode : " + dest);
str.WriteLine("Message : " + messageIn);
str.WriteLine("Operator :" + operatorNew);
str.Flush();
}
}
}
Let's say your console app project name is DataPrep. Now you can write in Data directory location by creating a new file namely as db.json
string filePath = Path.GetFullPath(Path.Combine(Environment.CurrentDirectory, #"..\..\..\")) + #"Data\db.json";
if (!File.Exists(filePath))
{
using (StreamWriter _streamWriter = File.CreateText(filePath))
{
_streamWriter.Write(resultAll);
_streamWriter.Flush();
}
}
else if (File.Exists(filePath))
{
using (var _streamWriter = new StreamWriter(filePath))
{
_streamWriter.Write(resultAll);
_streamWriter.Flush();
}
}