We Keep Coding

XML Serialization leaves file blank after restart

We have a problem where our industrial equipments software's .XML settings files become blank, yet they still have the correct number of bytes.
I have a feeling it might be caused by the way the customers are shutting down the PC as it tends to happen after they've down a shutdown, isolate, and boot. The way I save the files is,
Serialize to %temp% file
Validate that the newly created file starts with <?xml
If the /backup folders version of the file is older than a day, copy the existing file to the /backup folder
Copy new file to overwrite existing file.
I thought maybe it's related to encoding, disk caching, Windows Update, or Windows Recovery.
Looking for ideas as I've spent two years chasing down why this is happening.
As per request, here is the code.
public static bool SerializeObjXml(object Object2Serialize, string FilePath, Type type, bool gzip = false)
{
if (!Path.IsPathRooted(FilePath))
FilePath = Path.Combine(ApplicationDir, FilePath);
bool isSuccess = false;
var tmpFile = Path.GetTempFileName();
try
{
for (int i = 0; i < 3; i++)
{
try
{
Directory.CreateDirectory(Path.GetDirectoryName(FilePath));
if (gzip)
{
using (var ms = new MemoryStream())
{
XmlSerializer bf = new XmlSerializer(type);
bf.Serialize(ms, Object2Serialize);
ms.Position = 0;
using (var fileStream = new BinaryWriter(File.Open(tmpFile, FileMode.Create)))
{
using (GZipStream gzipStream = new GZipStream(fileStream.BaseStream, CompressionMode.Compress))
{
byte[] buffer = new byte[4096];
int numRead;
while ((numRead = ms.Read(buffer, 0, buffer.Length)) != 0)
{
gzipStream.Write(buffer, 0, numRead);
}
}
}
}
if (!FileChecker.isGZip(tmpFile))
throw new XmlException("Failed to write valid XML file " + FilePath);
}
else
{
using (var fs = new StreamWriter(File.Open(tmpFile, FileMode.Create), Encoding.UTF8))
{
XmlSerializer bf = new XmlSerializer(type);
bf.Serialize(fs, Object2Serialize);
}
if (!FileChecker.isXML(tmpFile))
throw new XmlException("Failed to write valid XML file " + FilePath);
}
isSuccess = true;
return true;
}
catch (XmlException)
{
return false;
}
catch (System.IO.DriveNotFoundException) { continue; }
catch (System.IO.DirectoryNotFoundException) { continue; }
catch (System.IO.FileNotFoundException) { continue; }
catch (System.IO.IOException) { continue; }
}
}
finally
{
if (isSuccess)
{
lock (FilePath)
{
try
{
//Delete existing .bak file
if (File.Exists(FilePath + ".bak"))
{
File.SetAttributes(FilePath + ".bak", FileAttributes.Normal);
File.Delete(FilePath + ".bak");
}
}
catch { }
try
{
//Make copy of file as .bak
if (File.Exists(FilePath))
{
File.SetAttributes(FilePath, FileAttributes.Normal);
File.Copy(FilePath, FilePath + ".bak", true);
}
}
catch { }
try
{
//Copy the temp file to the target
File.Copy(tmpFile, FilePath, true);
//Delete .bak file if no error
if (File.Exists(FilePath + ".bak"))
File.Delete(FilePath + ".bak");
}
catch { }
}
}
try
{
//Delete the %temp% file
if (File.Exists(tmpFile))
File.Delete(tmpFile);
}
catch { }
}
return false;
}
public static class FileChecker
{
const string gzipSig = "1F-8B-08";
static string xmlSig = "EF-BB-BF";// <?x";
public static bool isGZip(string filepath)
{
return FileChecker.CheckSignature(filepath, (3, gzipSig)) != null;
}
public static bool isXML(string filepath)
{
return FileChecker.CheckSignature(filepath, (3, xmlSig)) != null;
}
public static bool isGZipOrXML(string filepath, out bool isGZip, out bool isXML)
{
var sig = FileChecker.CheckSignature(filepath, (3, gzipSig), (3, xmlSig));
isXML = (sig == xmlSig);
isGZip = (sig == gzipSig);
return isXML || isGZip;
}
public static string CheckSignature(string filepath, params (int signatureSize, string expectedSignature)[] pairs)
{
if (String.IsNullOrEmpty(filepath))
throw new ArgumentException("Must specify a filepath");
if (String.IsNullOrEmpty(pairs[0].expectedSignature))
throw new ArgumentException("Must specify a value for the expected file signature");
int signatureSize = 0;
foreach (var pair in pairs)
if (pair.signatureSize > signatureSize)
signatureSize = pair.signatureSize;
using (FileStream fs = new FileStream(filepath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
if (fs.Length < signatureSize)
return null;
byte[] signature = new byte[signatureSize];
int bytesRequired = signatureSize;
int index = 0;
while (bytesRequired > 0)
{
int bytesRead = fs.Read(signature, index, bytesRequired);
bytesRequired -= bytesRead;
index += bytesRead;
}
foreach (var pair in pairs)
{
string actualSignature = BitConverter.ToString(signature, 0, pair.signatureSize);
if (actualSignature == pair.expectedSignature)
return actualSignature;
}
}
return null;
}
}

Using the operating system's move or copy file to overwrite an existing file is an atomic operation meaning the it wholly succeeds or doesn't and doesn't overlap other file operations.
Therefore what you have should work if that is how you are achieving step 4.
Copy new file to overwrite existing file.
If instead you are blanking out the existing file and re-writing the data I suspect that could be the the point of failure..

The issues while file space is being allocated the write is not occurring during shutdown, which leaves you when a file with bytes allocated without the data being flushed to disk.
During the OS shutdown, likely a ThreadAbortException is raised which triggers your finally block.
You can attempt to reproduce by calling Process.Start("shutdown", "-a") before your return statement but after you have set success = true.
I would suggest simplifying your code and have everything run inside of your try {} statement. This removes the possibility of having a state where success = true before your attempted your write to disk, which is then triggered in a finally statement trigged by a windows shutdown.
public static bool SerializeObjXml(
object Object2Serialize,
string FilePath,
Type type,
bool gzip = false)
{
if (!Path.IsPathRooted(FilePath))
FilePath = Path.Combine(ApplicationDir, FilePath);
Directory.CreateDirectory(FilePath);
for (int i = 0; i < 3; i++)
{
try
{
var tempFi = SerializeToXmlFile(Object2Serialize, type, gzip);
var fi = new FileInfo(FilePath);
if (fi.Exists)
fi.CopyTo(fi.FullName + ".bak", true);
tempFi.CopyTo(fi.FullName, true);
tempFi.Delete();
return true;
}
catch (Exception ex)
{
string message = $"[{DateTime.Now}] Error serializing file {FilePath}. {ex}";
File.WriteAllText(FilePath + ".log", message);
}
}
return false;
}
As a side note, you can simply use [Stream.CopyTo][1] and write directly to your temp file, without the need for intermediary streams or for manual buffer/byte read/write operations:
private static FileInfo SerializeToXmlFile(
object Object2Serialize,
Type type,
bool gzip)
{
var tmpFile = Path.GetTempFileName();
var tempFi = new FileInfo(tmpFile);
if (!gzip)
{
using (var fs = File.Open(tmpFile, FileMode.Create))
(new XmlSerializer(type)).Serialize(fs, Object2Serialize);
if (!FileChecker.isXML(tmpFile))
throw new Exception($"Failed to write valid XML file: {tmpFile}");
}
else
{
using (var fs = File.Open(tmpFile, FileMode.CreateNew))
using (var gz = new GZipStream(fs, CompressionMode.Compress))
(new XmlSerializer(type)).Serialize(fs, Object2Serialize);
if (!FileChecker.isGZip(tmpFile))
throw new Exception($"Failed to write valid XML gz file: {tmpFile}");
}
return tempFi;
}

Out of memory exception even for a very small task

I have a TCP application where I can request images from a folder on the server from the client. If I request a small folder, it works fine. If its a big folder it will throw an out of memory exception. But then anything after that, even a folder with 1 file will throw the same out of memory exception.
I thought it might have been the thread that is out of memory, so I tried to put it on a separate thread and task but neither worked. Here is the code I'm using:
public static void Images(string path)
{
new Task(() =>
{
try
{
string root = lookupDirectoryPath("Application data");
string backupPath = root + #"\Apple Computer\MobileSync\";
string imagePath = backupPath + path;
if (Directory.Exists(imagePath))
{
String[] allfiles = Directory.GetFiles(imagePath, "*.*", SearchOption.AllDirectories);
List<Image> allImages = new List<Image>();
foreach (string file in allfiles)
{
using (FileStream stream = new FileStream(file, FileMode.Open, FileAccess.Read))
{
if (IsImage(stream))
{
allImages.Add(Image.FromFile(file));
}
}
}
if (allImages.Count > 0)
{
byte[] data = imageListToByteArray(allImages);
serverSendByteArray(data, 12);
}
else
{
serverSendByteArray(Encoding.Default.GetBytes("backup contained no images"), 1);
}
}
else
{
serverSendByteArray(Encoding.Default.GetBytes("iphone backup folder does not exist"), 1);
}
}
catch (Exception ex)
{
if (ex.GetType().IsAssignableFrom(typeof(OutOfMemoryException)))
{
serverSendByteArray(Encoding.Default.GetBytes("Out of memory, could not send iphone images"), 1);
}
else
{
serverSendByteArray(Encoding.Default.GetBytes("Unknown error, could not send iphone images"), 1);
}
}
}).Start();
}
The exception gets thrown at allImages.Add(Image.FromFile(file));
this is the isImage() function:
public static bool IsImage(Stream stream)
{
stream.Seek(0, SeekOrigin.Begin);
List<string> jpg = new List<string> { "FF", "D8" };
List<string> bmp = new List<string> { "42", "4D" };
List<string> gif = new List<string> { "47", "49", "46" };
List<string> png = new List<string> { "89", "50", "4E", "47", "0D", "0A", "1A", "0A" };
List<List<string>> imgTypes = new List<List<string>> { jpg, bmp, gif, png };
List<string> bytesIterated = new List<string>();
for (int i = 0; i < 8; i++)
{
string bit = stream.ReadByte().ToString("X2");
bytesIterated.Add(bit);
bool isImage = imgTypes.Any(img => !img.Except(bytesIterated).Any());
if (isImage)
{
return true;
}
}
return false;
}
Thanks for any help

I tried and I can reproduce your problem. It is definitely out of memory and nothing like "It just seems to be" the memory usage increases to about 4 GB and then the error shows up. Console output is just to see what's happening there.
The Image object seems to be not the best way to save the data.
I tried this and got this to work with many many files. Maybe you can change the code to fit your needs:
String[] allfiles = Directory.GetFiles(imagePath, "*.*", SearchOption.AllDirectories);
//List<Image> allImages = new List<Image>();
List<Byte[]> allImagesBytes = new List<Byte[]>();
foreach (string file in allfiles)
{
using (FileStream stream = new FileStream(file, FileMode.Open, FileAccess.Read))
{
if (IsImage(stream))
{
Console.Clear();
Console.Write(allImagesBytes.Count());
//allImages.Add(Image.FromStream(stream));
//allImages.Add(Image.FromFile(file));
allImagesBytes.Add(File.ReadAllBytes(file));
}
}
}

Image.FromFile seem to cause the error. In the following question it was a corrupeted image file or running out of file handles, Image.FromStream() did it better. Worth a try cause you already have the stream open:
https://stackoverflow.com/a/2216338/7803013

Try changing this
using (FileStream stream = new FileStream(file, FileMode.Open, FileAccess.Read))
{
if (IsImage(stream))
{
allImages.Add(Image.FromFile(file));
}
}
...
if (allImages.Count > 0)
{
byte[] data = imageListToByteArray(allImages);
serverSendByteArray(data, 12);
}
into this:
using (FileStream stream = new FileStream(file, FileMode.Open, FileAccess.Read))
{
if (IsImage(stream))
{
allImages.Add(Image.FromFile(file));
}
stream.Close();
}
....
if (allImages.Count > 0)
{
byte[] data = imageListToByteArray(allImages);
foreach(Image img in allImages)
{
img.Dispose();
}
serverSendByteArray(data, 12);
}

Using Renci ssh to upload to SFTP directory

I am not getting an error but my files are not getting uploaded. Am trying to upload to a targetDirectory on SFTP.
public string TryUploads(string targetDirectory)
{
string _localDirectory = LocalDirectory; //The directory in SFTP server where the files are present
if (oSftp == null)
{
oSftp = Instance;
}
lock (thisLock)
{
try
{
oSftp.Connect();
List<string> fileList = Directory.GetFiles(_localDirectory, "*.*").ToList<string>();
oSftp.ChangeDirectory(targetDirectory);
if (fileList != null && fileList.Count() > 1)
{
for (int i = 0; i < fileList.Count(); i++)
{
string ftpFileName = Path.GetFileName(fileList[i]);
if (!String.IsNullOrEmpty(targetDirectory))
ftpFileName = String.Format("{0}/{1}", targetDirectory, ftpFileName);
using (var stream = new FileStream(Path.GetFileName(fileList[i]), FileMode.Create))
{
oSftp.BufferSize = 4 * 1024;
oSftp.UploadFile(stream, ftpFileName);
// stream.Close();
}
}
}
oSftp.Disconnect();
}
catch (Exception e)
{
throw new ApplicationException(e.Message);
}
}
return Strings.StatusOk;
}

I solved the issue. I needed to put the location of where am getting the file from in the stream
using (var stream = new FileStream(LocalDirectory + "\\" + Path.GetFileName(fileList[i]), FileMode.Open))
Solved. Files uploaded ;)

Why does this Sharepoint Files.Add() result in an I/O Error?

With this code:
using (var ms = new MemoryStream())
{
using (var doc = new Document(PageSize.A4, 25, 25, 10, 10))
{
//Create a writer that's bound to our PDF abstraction and our
stream
using (var writer = PdfWriter.GetInstance(doc, ms))
{
//Open the document for writing
doc.Open();
. . .
}
// File has been generated, now save it
try
{
var bytes = ms.ToArray();
String pdfFileID = GetYYYYMMDDAndUserNameAndAmount();
String pdfFileName = String.Format("DirectPayDynamic_{0}.pdf",
pdfFileID);
String fileFullpath =
Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.DesktopDirector
y), pdfFileName);
String fileLinkBase = "Generated PDF: {1}";
String filelink = String.Format(fileLinkBase, fileFullpath,
pdfFileName);
File.WriteAllBytes(fileFullpath, bytes);
AddVerticalSpace();
var pdflink = new Label
{
CssClass = "finaff-webform-field-label",
Text = filelink
};
this.Controls.Add(pdflink);
// NOTE: This is the new (non-working, exception-throwing) part of the
code, where we're trying to save the PDF file to a Sharepoint Document Library
string destination = String.Format("DirectPayPDFForms/{0}",
pdfFileName);
SPSite siteCollection = new SPSite(siteUrl);
SPWeb site = SPContext.Current.Web;
site.Files.Add(destination, ms); // this is the line that fails
// end of new code
}
catch (DocumentException dex)
{
exMsg = dex.Message;
}
catch (IOException ioex)
{
exMsg = ioex.Message;
}
catch (Exception ex)
{
exMsg = ex.Message;
; // for debugging: What is "ex" here?
}
} // using (var ms = new MemoryStream())
} // GeneratePDF
...I get, "I/O Error Occurred" (in the IOException catch block). It is this line:
site.Files.Add(destination, ms);
...that throws the error. Is it my destination, or the memory stream, that is causing the problem? "destination" is the name of the Sharepoint Document Library (DirectPayPDFForms) plus a generated name for the file. Without this new code, the method runs fine and places the PDF file on the server (but that's not where we want it - we need it to go first to a Document Library).
UPDATE
I get the same exception replacing the problematic code block above with a call to this:
private void SavePDFToDocumentLibrary(MemoryStream memstrm)
{
string doclib = "DirectPayPDFForms";
try
{
using (SPSite site = new SPSite(siteUrl))
{
using (SPWeb web = site.RootWeb)
{
SPList list = web.Lists[doclib];
SPListItem spli = list.Items.Add();
spli["Title"] = String.Format("DirectPayPDFForms-{0}-{1}", GetUserId(), GetListTitleTimeStamp());
if (null != memstrm)
{
web.Files.Add(doclib, memstrm);
}
// If this works, update at least some of the fields, such as Created, CreatedBy, etc.
spli.Update();
}
}
}
catch (Exception ex)
{
String s = String.Format("Exception is {0}", ex.Message);
}
}
This type of code works in another instance (I can successfully save values to a List this way), so the problem must be something Document-Library-specific. Do I need to save the document to a specific field in the Document Library?
UPDATE 2
I also tried this:
string saveloc = String.Format(#"{0}\{1}", doclib, filename);
...with the same result.
And this:
string saveloc = String.Format(#"{0}\{1}\{2}", siteUrl, doclib, filename);
...which at least provided some variety, exceptioning with, "Invalid URI: The hostname could not be parsed."
With this:
string saveloc = String.Format("{0}/{1}/{2}", siteUrl, doclib, filename);
...I'm back to the old "I/O Error Occurred" mantra.
UPDATE 3
Since the "AllItems.aspx" page for the Sharepoint site has both "Documents" and "Lists", I am wondering if I should not be using List in this case, but Document. IOW, perhaps my code should be something like:
SPDocTemplateCollection spdoctemplatecoll = web.DocTemplates;
SPDocumentLibrary spdoclib = spdoctemplatecoll[doclib];
SPListItem spli = spdoclib.Items.Add();
...where it is currently:
SPList list = web.Lists[doclib];
SPListItem spli = list.Items.Add();
SPListItem spli = list.Items.Add();
...but that guess misses fire, as it won't compile (I get, "The best overloaded method match for 'Microsoft.SharePoint.SPDocTemplateCollection.this[int]' has some invalid arguments" and "Argument 1: cannot convert from 'string' to 'int'")

This works:
. . .
SavePDFToDocumentLibrary(fileFullpath); // instead of trying to send the memory stream, using the file saved on the server
. . .
private void SavePDFToDocumentLibrary(String fullpath)
{
String fileToUpload = fullpath;
String sharePointSite = siteUrl;
String documentLibraryName = "DirectPayPDFForms";
using (SPSite oSite = new SPSite(sharePointSite))
{
using (SPWeb oWeb = oSite.OpenWeb())
{
if (!System.IO.File.Exists(fileToUpload))
{
throw new FileNotFoundException("File not found.", fileToUpload);
}
SPFolder doclib = oWeb.Folders[documentLibraryName];
// Prepare to upload
Boolean replaceExistingFiles = true;
String fileName = System.IO.Path.GetFileName(fileToUpload);
FileStream fileStream = File.OpenRead(fileToUpload);
// Upload document
SPFile spfile = doclib.Files.Add(fileName, fileStream, replaceExistingFiles);
// Commit
doclib.Update();
}
}
}
I adapted it from Henry Zucchini's answer here.

Memorystream object being passed to method doesn't contain values

I am using a memorystream object to write values from streamwriter object into memory. I have several methods which are for logging errors and values so I pass my memorystream object between them as a parameter.
My problem is the memorystream doesn't contain values. See code below:
#region csvlogging
public static void initiateCsvLogging(SortedList<int, string> logList, SortedList<int, string> errorList) // Handles csv upload logging
{
logMsgError(errorList, logList);
}
public static void logMsgError(SortedList<int, string> errorList, SortedList<int, string> logList)
{
using (MemoryStream ms = new MemoryStream())
{
StreamWriter sw = new StreamWriter(ms);
try
{
sw.WriteLine("Errors:");
foreach (KeyValuePair<int, string> k in errorList)
{
if (k.Value == null)
{
sw.WriteLine("No errors reported.");
}
else
{
sw.WriteLine(k.Key + "," + k.Value);
}
}
}
catch (Exception ex)
{
}
finally
{
sw.WriteLine("");
}
logMsg(logList, ms);
}
} // Handles data upload error logging for csv
public static MemoryStream logMsg(SortedList<int, string> logList, MemoryStream ms)
{
string DateNow = System.DateTime.Now.Date.ToString("yyyy-MM-dd");
string FileName = "log " + DateNow + ".csv";
char[] delimiterChars = { ',' };
try
{
// Write values to textfile and save to Log folder
using (StreamWriter sw = new StreamWriter(ms))
{
if (logList.Keys.Count == 0)
{
sw.WriteLine("No new users added to Active Directory.");
}
else // If keys > 0 then new users have been added to AD
{
sw.WriteLine("Username" + "," + "Password" + "," + "Company" + "," + "Email");
foreach (KeyValuePair<int, string> k in logList)
{
string[] values = k.Value.Split(delimiterChars);
sw.WriteLine(values[0] + "," + values[1] + "," + values[2] + "," + values[3]);
}
}
try
{
sw.Flush();
sendLogByEmail(new MemoryStream(ms.ToArray()), FileName);
}
catch (Exception ex)
{
}
}
}
catch (ThreadAbortException ex)
{
}
finally
{
}
return ms;
} // Handles logging for csv
public static void sendLogByEmail(MemoryStream ms, string error, int count)
{
//Send mail by attachment code
SmtpClient smtpclient = new SmtpClient();
//smtpclient.DeliveryMethod = SmtpDeliveryMethod.PickupDirectoryFromIis;
//smtpclient.UseDefaultCredentials = true;
smtpclient.Host = "ldnmail";
MailMessage message = new MailMessage("nick.gowdy#orcinternational.co.uk", "nick.gowdy#orcinternational.co.uk");
System.Net.Mime.ContentType ct = new System.Net.Mime.ContentType(System.Net.Mime.MediaTypeNames.Text.Plain);
System.Net.Mail.Attachment attach = new System.Net.Mail.Attachment(ms, ct);
attach.ContentDisposition.FileName = error;
message.Attachments.Add(attach);
smtpclient.Send(message);
}
#endregion
Because I am not using the USING keyword do I have to write some more code for the memorystream object?

You have to flush (Close) the StreamWriter.
finally
{
sw.WriteLine("");
}
sw.Flush();
logMsg(logList, ms);
But in the end you use all data to send it by mail:
System.Net.Mime.ContentType ct = new
System.Net.Mime.ContentType(System.Net.Mime.MediaTypeNames.Text.Plain);
System.Net.Mail.Attachment attach = new System.Net.Mail.Attachment(ms, ct);
It would seem a lot easier to collect the information in a StringBuilder through an attached StringWriter.
There are issues with a StreamWriter closing its Stream, that may be part of your problem.