We Keep Coding

How to detect if a File buffer has been flushed

I'm trying to create a simple logging tool to monitor file changes. I have used the FileSystemWatcher to detect changes to the file, but I've discovered that the events are only triggered when the file is closed, not when the buffer is flushed. This means that if multiple lines are added before the file is closed, I will only see that when the file is closed.
here is my test example.
[TestClass]
public class FileWriteTests
{
[TestMethod]
public void TestMethodAfterClose()
{
var currentDir = Environment.CurrentDirectory;
var fileToMonitor = "test.txt";
List<string> output = new List<string>();
var watcherTest = new FileWatcherTest(fileToMonitor, currentDir, output);
File.Delete(Path.Combine(currentDir, fileToMonitor));
using (var writer = new StreamWriter(Path.Combine(currentDir, fileToMonitor), true))
{
writer.WriteLine($"test");
writer.Flush();
}
System.Threading.Thread.Sleep(10);
Assert.AreEqual(1, output.Count);
Assert.AreEqual("test", output[0]);
}
[TestMethod]
public void TestMethodAfterFlush()
{
var currentDir = Environment.CurrentDirectory;
var fileToMonitor = "test.txt";
List<string> output = new List<string>();
var watcherTest = new FileWatcherTest(fileToMonitor, currentDir, output);
File.Delete(Path.Combine(currentDir, fileToMonitor));
using (var writer = new StreamWriter(Path.Combine(currentDir, fileToMonitor), true))
{
try
{
writer.WriteLine($"test");
writer.Flush();
System.Threading.Thread.Sleep(1000);
// add break point here for BareTail
Assert.AreEqual(1, output.Count);
Assert.AreEqual("test", output[0]);
}
catch
{
Assert.Fail("Test failed");
}
}
}
public class FileWatcherTest
{
public string FileName { get; set; }
public string Directory { get; set; }
private List<string> linesRead;
private FileSystemWatcher watcher;
public FileWatcherTest(string fileName, string directory, List<string> output)
{
FileName = fileName;
Directory = directory;
linesRead = output;
watcher = new FileSystemWatcher();
watcher.Path = directory;
watcher.Filter = FileName;
watcher.Changed += Watcher_Changed;
watcher.EnableRaisingEvents = true;
watcher.NotifyFilter = NotifyFilters.Attributes |
NotifyFilters.CreationTime |
NotifyFilters.DirectoryName |
NotifyFilters.FileName |
NotifyFilters.LastAccess |
NotifyFilters.LastWrite |
NotifyFilters.Security |
NotifyFilters.Size;
}
private void Watcher_Changed(object sender, FileSystemEventArgs e)
{
using (var fileStream = File.Open(Path.Combine(Directory, FileName), FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete | FileShare.Inheritable))
{
using (var reader = new StreamReader(fileStream))
{
string line;
while ((line = reader.ReadLine()) != null)
{
linesRead.Add(line);
}
}
}
}
}
}
right now TestMethodAfterClose succeeds and TestMethodAfterFlush fails. When I use the program BareTail and wait at the breakpoint, I see that it updates the display before the file is closed. So that gives me an indication that it's possible. I don't know if it's possible in C# and I might need to import some native functions using dllimport. The problem is I don't know where to look
How do I make both tests succeed, while not using a timer?
EDIT:
updated the FileWatcherTest class

Unfortunately Flush doesn't flush the thing you want. I find a lot articles to explain it, for example:
https://blogs.msdn.microsoft.com/alejacma/2011/03/23/filesystemwatcher-class-does-not-fire-change-events-when-notifyfilters-size-is-used/
There is a solution since .net 4, use another overload method of FileStream: Flush(bool)
var fs = writer.BaseStream as FileStream;
fs.Flush(true);
And you only give disk 10ms to react, maybe this is another problem.

After some searching, I discovered that the FileSystemWatcher only triggers an event after a file is closed as shown in this article. The article only mentions the date modified NotifyFilter, but in my testing I found that all Notifyfilters trigger after the file is closed and never while it's still open.
For that reason, it looks like tailing a file is only possible with a looped function that continuously monitors the file for extra rows. I used the code on this link as an example.
Here is my code working:
[TestClass]
public class FileWriteTests
{
[TestMethod]
public void TestMethodAfterClose_filetailing()
{
var currentDir = Environment.CurrentDirectory;
var fileToMonitor = "test.txt";
File.Delete(Path.Combine(currentDir, fileToMonitor));
List<string> output = new List<string>();
using (var watcherTest = new PersonalFileTail(currentDir, fileToMonitor))
{
watcherTest.StartTail(delegate (string line) { output.Add(line); });
using (var writer = new StreamWriter(Path.Combine(currentDir, fileToMonitor), true))
{
writer.WriteLine($"test");
writer.Flush();
}
System.Threading.Thread.Sleep(200);
watcherTest.StopTail();
}
System.Threading.Thread.Sleep(10);
Assert.AreEqual(1, output.Count);
Assert.AreEqual("test", output[0]);
}
[TestMethod]
public void TestMethodAfterFlush_filetailing()
{
// initiate file
var currentDir = Environment.CurrentDirectory;
var fileToMonitor = "test.txt";
File.Delete(Path.Combine(currentDir, fileToMonitor));
FileInfo info = new FileInfo(Path.Combine(currentDir, fileToMonitor));
List<string> output = new List<string>();
using (var watcherTest = new PersonalFileTail(currentDir, fileToMonitor))
{
watcherTest.StartTail(delegate (string line) { output.Add(line); });
using (var writer = new StreamWriter(Path.Combine(currentDir, fileToMonitor), true))
{
try
{
writer.WriteLine($"test");
writer.Flush();
System.Threading.Thread.Sleep(1000);
Assert.AreEqual(1, output.Count);
Assert.AreEqual("test", output[0]);
}
catch
{
Assert.Fail("Test failed");
}
}
watcherTest.StopTail();
}
}
public class PersonalFileTail : IDisposable
{
private string filename;
private string directory;
private Task fileTailTask;
private Action<string> handleResults;
private volatile bool runTask;
private long lastFilePosition;
public string FileName
{
get { return Path.Combine(directory, filename); }
}
public PersonalFileTail(string directory, string filename)
{
this.directory = directory;
this.filename = filename;
this.runTask = false;
lastFilePosition = 0;
}
public void StartTail(Action<string> handleResults)
{
this.handleResults = handleResults;
runTask = true;
fileTailTask = Task.Run(() => MonitorFileTask());
}
public void StopTail()
{
runTask = false;
fileTailTask.Wait();
}
public IEnumerable<string> ReadLinesFromFile()
{
using (StreamReader reader = new StreamReader(new FileStream(FileName,
FileMode.Open, FileAccess.Read, FileShare.ReadWrite)))
{
string line = "";
while ((line = reader.ReadLine()) != null)
{
yield return line;
}
lastFilePosition = reader.BaseStream.Length;
}
}
public void MonitorFileTask()
{
StreamReader reader = null;
FileStream stream = null;
try
{
using(stream = new FileStream(FileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
using (reader = new StreamReader(stream))
{
do
{
//if the file size has increased do something
if (reader.BaseStream.Length > lastFilePosition)
{
//seek to the last max offset
reader.BaseStream.Seek(lastFilePosition, SeekOrigin.Begin);
//read out of the file until the EOF
string line = "";
while ((line = reader.ReadLine()) != null)
{
handleResults(line);
}
//update the last max offset
lastFilePosition = reader.BaseStream.Position;
}
// sleep task for 100 ms
System.Threading.Thread.Sleep(100);
}
while (runTask);
}
}
catch
{
if (reader != null)
reader.Dispose();
if (stream != null)
stream.Dispose();
}
}
public void Dispose()
{
if(runTask)
{
runTask = false;
fileTailTask.Wait();
}
}
}
}
If anyone knows a way in which tailing can be done without using a timed function, I will accept that as the answer. Until that time, i feel that my answer is the only possible way to do this.

Set position StreamReader doesn't work read append-only text file

I have a append-only log file which is monitored by a FileSystemWatcher. When the file is changed, Read() is called for the LogFile object.
The log file should be read line by line.
The goal is to read only changes i.e. lines add to the log file (skip already readed lines).
Thus the StreamReader should start to read from the position where it ended on the previous read.
My solution so far doesn't work. When I add
1
2
3
4
line by line in Notepad++ to my textfile & save each time when a line was added, the Debug output is
Initial read
1 //OK
2 //OK
3 //OK
1 //looks like the log file is read again from the beginning
2
3
4
Output should be
Initial read
1
2
3
4
Any ideas to solve this problem?
Console code
public class LogFile
{
public List<string> Contents { get; }
string _fullPath;
long position;
public LogFile(string fullPath)
{
if (File.Exists(fullPath))
{
_fullPath = fullPath;
Contents = new List<string>();
Read();
}
else
{
throw new FileNotFoundException($"{fullPath} not found");
}
}
public void Read(FileSystemWatcher fsw = null)
{
if (fsw != null)
fsw.EnableRaisingEvents = false; //set to false to prevent Changed event be fired twice
FileStream fs = null;
StreamReader sr = null;
try
{
fs = new FileStream(_fullPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
try
{
sr = new StreamReader(fs, Encoding.UTF8);
if (Contents.Count == 0)
{
Debug.WriteLine($"Initial read");
AddToContents(_fullPath, sr);
position = fs.Length; //store the length of the stream
}
else
{
sr.DiscardBufferedData();
sr.BaseStream.Seek(position, SeekOrigin.Begin);
AddToContents(_fullPath, sr);
position = fs.Length; //store the length of the stream
}
}
catch (Exception ex)
{
Debug.WriteLine($"Error while reading from {_fullPath}");
//log exception
}
finally
{
if (sr != null)
sr.Close();
}
}
catch (Exception ex)
{
Debug.WriteLine($"Error while opening {_fullPath}");
//log exception
}
finally
{
if (fs != null)
fs.Close();
if (fsw != null)
fsw.EnableRaisingEvents = true; //set raise events for the filesystemwatcher to true
}
}
private List<string> AddToContents(string fullPath, StreamReader sr)
{
List<string> newLines = new List<string>();
try
{
while (!sr.EndOfStream)
{
try
{
string line = sr.ReadLine();
if (line != string.Empty)
{
Contents.Add(line);
newLines.Add(line);
Debug.WriteLine(line);
}
}
catch (Exception ex)
{
Debug.WriteLine($"Error processing line in {fullPath}");
throw;
}
}
return newLines;
}
catch (Exception ex)
{
Debug.WriteLine($"Error while reading from {fullPath}");
throw;
}
}
}
class Program
{
static LogFile file;
static FileSystemWatcher fsw;
static void Main(string[] args)
{
string path = #"C:\Temp\test.txt";
file = new LogFile(path);
CreateWatcher(path);
Console.ReadKey();
}
private static FileSystemWatcher CreateWatcher(string fileNameFullPath)
{
fsw = new FileSystemWatcher(Path.GetDirectoryName(fileNameFullPath)); //constructor only accepts directory path
fsw.IncludeSubdirectories = false;
fsw.Filter = Path.GetFileName(fileNameFullPath); //filter for the given file
fsw.NotifyFilter = NotifyFilters.LastWrite;
fsw.EnableRaisingEvents = true;
fsw.Changed += Fsw_Changed;
return fsw;
}
private static void Fsw_Changed(object sender, FileSystemEventArgs e)
{
if (file != null)
file.Read(fsw);
}
}

Problem is
position = fs.Length; //store the length of the stream
You should store current position of the stream into position field not length of stream because sometimes FileStream.Length is zero (I don't know why)
this.position = fs.Position;
and check if FileStream.Length is zero skip that change
fs = new FileStream(this._fullPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
if (fs.Length != 0)
{
try
{
sr = new StreamReader(fs);
if (this.Contents.Count == 0)
{
......
Now it's working

MS Cognitive Face API: Unable to detect faces

I am trying to use Microsoft Cognitive Face API for the first time. Documentation gives quite a simple method to detect face from memory stream. I am trying to detect faces from images located inside a folder. Right now there is only one image inside the folder. The issue is whenever the control reaches the following line:
var faces = await faceServiceClient.DetectAsync(memStream, true, true);
it terminates without any exception or error. Here is the complete code I have written.
using Microsoft.ProjectOxford.Face;
using Microsoft.ProjectOxford.Common;
using Microsoft.ProjectOxford.Face.Contract;
using System;
using System.Collections.Generic;
using System.Configuration;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
namespace FaceDetection.FaceDetect
{
class Program
{
static void Main(string[] args)
{
Console.Title = "Face Detect";
Start();
}
static async Task Stop()
{
await Close();
}
private static Task Close()
{
return Task.Run(() =>
{
Environment.Exit(0);
});
}
static async Task ReStart(string _reason = "")
{
Console.WriteLine(_reason + "To restart the process press 'R'. To exit press 'X'");
var _response = Console.ReadLine();
if (_response == "r" || _response == "R")
await Start();
else
await Stop();
}
static async Task Start()
{
Console.Clear();
Console.WriteLine("Enter Folder Path");
string imageFolderPath = Console.ReadLine();
if (!Directory.Exists(imageFolderPath))
{
await ReStart("Folder does not exist! ");
}
else
{
await SaveFiles(imageFolderPath);
}
}
static async Task SaveFiles(string imageFolderPath)
{
try
{
DirectoryInfo dInfo = new DirectoryInfo(imageFolderPath);
string[] extensions = new[] { ".jpg", ".jpeg" };
FileInfo[] files = dInfo.GetFiles()
.Where(f => extensions.Contains(f.Extension.ToLower()))
.ToArray();
if (files.Length == 0)
await ReStart("No files found in the specified folder! ");
else
{
string subscriptionKey = "ADSFASDFASDFASDFASDFASDFASDF";
if (!String.IsNullOrEmpty(ConfigurationManager.AppSettings["subscriptionKey"]))
subscriptionKey = ConfigurationManager.AppSettings["subscriptionKey"].ToString();
//var stringFaceAttributeType = new List<FaceAttributeType> { FaceAttributeType.Smile, FaceAttributeType.Glasses, FaceAttributeType.Gender, FaceAttributeType.Age };
//IEnumerable<FaceAttributeType> returnFaceAttributes = stringFaceAttributeType;
IFaceServiceClient faceServiceClient = new FaceServiceClient(subscriptionKey);
foreach (FileInfo file in files)
{
try
{
using (FileStream fileStream = File.OpenRead(imageFolderPath + "\\" + file.Name))
{
MemoryStream memStream = new MemoryStream();
memStream.SetLength(fileStream.Length);
fileStream.Read(memStream.GetBuffer(), 0, (int)fileStream.Length);
//Used following commented code to make sure MemoryStream is not corrupted.
//FileStream _file = new FileStream(imageFolderPath + "\\test.jpg", FileMode.Create, FileAccess.Write);
//memStream.WriteTo(_file);
//_file.Close();
//memStream.Close();
try
{
//This line never returns a result. The execution terminates without any exception/error.
var faces = await faceServiceClient.DetectAsync(memStream, true, true);
if (faces != null)
{
foreach (var face in faces)
{
var rect = face.FaceRectangle;
var landmarks = face.FaceLandmarks;
}
}
else
Console.WriteLine("No face found in image: " + file.FullName);
}
catch (Exception ex)
{
Console.WriteLine("Error");
}
}
}
catch (Exception ex)
{
Console.WriteLine("There was an error!");
}
}
}
}
catch (Exception ex)
{
Console.WriteLine("There was an error!");
}
await ReStart();
}
}
}
Can someone point out what am I missing. Why is this code not working?

When you read the file in to the MemoryStream, your read pointer is advanced to the end. So memStream passed in to DetectAsync() appears empty. The fact is you need not copy your file to memory. You could simply pass in the FileStream after opening.
using (FileStream fileStream = File.OpenRead(imageFolderPath + "\\" + file.Name))
{
try
{
var faces = await faceServiceClient.DetectAsync(fileStream, true, true);
if (faces != null)
{
foreach (var face in faces)
{
var rect = face.FaceRectangle;
var landmarks = face.FaceLandmarks;
}
}
else
{
Console.WriteLine("No face found in image: " + file.FullName);
}
catch (Exception ex)
{
Console.WriteLine("Error");
}
}
Alternatively, you can rewind the memory stream by setting memStream.Position = 0 before calling DetectAsync.

Windows Form XML Serilization Load Dialog

I've got a windows form with save/loading of XML files and it asks the user where they want to save/load it. My problem is I dont know how to change this method to load the file from where the user wants and not where the streamreader specifies.
The code below is of my button and LoadValues Method.
private void Edittask_loadbuttonClick(
object sender, EventArgs e)
{
Stream myStream = null;
var sFile1 = new OpenFileDialog();
sFile1.InitialDirectory = "c:\\";
sFile1.Filter = "xml files (*.xml)|*.xml";
sFile1.FilterIndex = 2;
sFile1.RestoreDirectory = true;
if (sFile1.ShowDialog() == DialogResult.OK)
{
try
{
if ((myStream = sFile1.OpenFile()) != null)
{
using (myStream)
{
var v = LoadValues();
this.load_task1_name.Text =
v.task1_name;
this.load_task1_desc.Text =
v.task1_desc;
this.load_task1_date.Value =
v.task1_date;
this.load_checkbox.Checked =
v.task1_checkbox;
}
}
}
catch (Exception ex)
{
MessageBox.Show(
"Error: Could not read file from disk. Original error: " +
ex.Message);
}
}
}
public Values LoadValues()
{
var serializer = new XmlSerializer(typeof (Values));
using (
TextReader textReader = new StreamReader(
"E:\\hello.xml")
)
{
return
(Values) serializer.Deserialize(textReader);
}
}

I would pass the Stream from the OpenFileDialog to LoadValues(...), and use that to construct your StreamReader:
public Values LoadValues(Stream stream)
{
XmlSerializer serializer = new XmlSerializer(typeof(Values));
using (TextReader textReader = new StreamReader(stream))
{
return (Values)serializer.Deserialize(textReader);
}
}
and
if ((myStream = sFile1.OpenFile()) != null)
{
using (myStream)
{
Values v = LoadValues(myStream);
...
}
}

You need to pass the Stream as a parameter to your function.

FileSystemWatcher not monitoring local user folder or temporary internet files folder in Vista (64bit)

I wrote a test program to monitor my Picture folder which points to c:\users[username]\Pictures and temporary internet files folder for the same user. This is program works perfectly fine if I change the folder to other location like d:\persona_pics.
Any idea why events are not being raised when I set the mentioned folder to monitor?
here is the code.
class Program
{
static void Main(string[] args)
{
//FileSystemWatcher myJpegFileWatcher = new FileSystemWatcher(#"C:\Users\[username]\AppData\Local\Microsoft\Windows\Temporary Internet Files\low\content.ie5\");
FileSystemWatcher myJpegFileWatcher = new FileSystemWatcher(#"C:\Users\[username]\Pictures\ ");
myJpegFileWatcher.Filter = "*.jpg";
myJpegFileWatcher.Created += new FileSystemEventHandler(myJpegFileWatcher_Created);
myJpegFileWatcher.Changed += new FileSystemEventHandler(myJpegFileWatcher_Changed);
myJpegFileWatcher.IncludeSubdirectories = true;
myJpegFileWatcher.NotifyFilter = NotifyFilters.CreationTime;
myJpegFileWatcher.EnableRaisingEvents = true;
Console.Read();
}
static void myJpegFileWatcher_Changed(object sender, FileSystemEventArgs e)
{
FileInfo duplicateFile = new FileInfo(#e.FullPath);
bool flag = true;
while (flag)
{
try
{
if (duplicateFile.Length > 20000)
{
duplicateFile.CopyTo(#"d:\pics\spy\ " + e.Name);
flag = false;
StreamWriter fs = new StreamWriter(#"d:\pics\log.txt", true);
fs.WriteLine("file is being copied:{0}, Size={1}", e.FullPath, duplicateFile.Length);
fs.Close();
}
else
{
StreamWriter fs = new StreamWriter(#"d:\pics\log.txt", true);
fs.WriteLine("file is not being copied:{0}, Size={1}", e.FullPath, duplicateFile.Length);
fs.Close();
}
}
catch (Exception ex)
{
//
}
}
}
static void myJpegFileWatcher_Created(object sender, FileSystemEventArgs e)
{
FileInfo duplicateFile = new FileInfo(#e.FullPath);
bool flag = true;
while (flag)
{
try
{
if (duplicateFile.Length > 20000)
{
duplicateFile.CopyTo(#"d:\pics\spy\ " + e.Name);
flag = false;
StreamWriter fs = new StreamWriter(#"d:\pics\log.txt", true);
fs.WriteLine("file is being copied:{0}, Size={1}", e.FullPath, duplicateFile.Length);
fs.Close();
}
}
catch (Exception ex)
{
//
}
}
}
}
Working code..
class Program
{
static void Main(string[] args)
{
FileSystemWatcher myJpegFileWatcher = new FileSystemWatcher(#"C:\Users\[user]\Pictures\");
myJpegFileWatcher.Filter = "*.jpg";
myJpegFileWatcher.Changed += new FileSystemEventHandler(myJpegFileWatcher_Changed);
myJpegFileWatcher.IncludeSubdirectories = true;
myJpegFileWatcher.EnableRaisingEvents = true;
Console.Read();
}
static void myJpegFileWatcher_Changed(object sender, FileSystemEventArgs e)
{
FileInfo duplicateFile = new FileInfo(#e.FullPath);
bool flag = true;
while (flag)
{
try
{
if (duplicateFile.Exists)
{
if (duplicateFile.Length > 20000)
{
try
{
duplicateFile.CopyTo(#"d:\pics\spy\" + e.Name,true);
}
catch (Exception ex)
{
StreamWriter fs = new StreamWriter(#"d:\pics\log.txt", true);
fs.WriteLine("Error Inside copying:{0}", ex.Message);
fs.Close();
}
finally
{
flag = false;
StreamWriter fs = new StreamWriter(#"d:\pics\log.txt", true);
fs.WriteLine("file is being copied:{0}, Size={1}", e.FullPath, duplicateFile.Length);
fs.Close();
}
}
else
{
StreamWriter fs = new StreamWriter(#"d:\pics\log.txt", true);
fs.WriteLine("file is not being copied:{0}, Size={1}", e.FullPath, duplicateFile.Length);
fs.Close();
}
}
}
catch (Exception ex)
{
StreamWriter fs = new StreamWriter(#"d:\pics\log.txt", true);
fs.WriteLine("Error:{0}", ex.Message);
fs.Close();
}
}
}
}

Try to run FileMon (SysInternals tool available through MSDN). It will show you what your code actually does on the file system. Then you might be able to find out why or what exactly behaves differently when you point your code to "My Pictures" etc.