I'm trying to create a simple logging tool to monitor file changes. I have used the FileSystemWatcher to detect changes to the file, but I've discovered that the events are only triggered when the file is closed, not when the buffer is flushed. This means that if multiple lines are added before the file is closed, I will only see that when the file is closed.
here is my test example.
[TestClass]
public class FileWriteTests
{
[TestMethod]
public void TestMethodAfterClose()
{
var currentDir = Environment.CurrentDirectory;
var fileToMonitor = "test.txt";
List<string> output = new List<string>();
var watcherTest = new FileWatcherTest(fileToMonitor, currentDir, output);
File.Delete(Path.Combine(currentDir, fileToMonitor));
using (var writer = new StreamWriter(Path.Combine(currentDir, fileToMonitor), true))
{
writer.WriteLine($"test");
writer.Flush();
}
System.Threading.Thread.Sleep(10);
Assert.AreEqual(1, output.Count);
Assert.AreEqual("test", output[0]);
}
[TestMethod]
public void TestMethodAfterFlush()
{
var currentDir = Environment.CurrentDirectory;
var fileToMonitor = "test.txt";
List<string> output = new List<string>();
var watcherTest = new FileWatcherTest(fileToMonitor, currentDir, output);
File.Delete(Path.Combine(currentDir, fileToMonitor));
using (var writer = new StreamWriter(Path.Combine(currentDir, fileToMonitor), true))
{
try
{
writer.WriteLine($"test");
writer.Flush();
System.Threading.Thread.Sleep(1000);
// add break point here for BareTail
Assert.AreEqual(1, output.Count);
Assert.AreEqual("test", output[0]);
}
catch
{
Assert.Fail("Test failed");
}
}
}
public class FileWatcherTest
{
public string FileName { get; set; }
public string Directory { get; set; }
private List<string> linesRead;
private FileSystemWatcher watcher;
public FileWatcherTest(string fileName, string directory, List<string> output)
{
FileName = fileName;
Directory = directory;
linesRead = output;
watcher = new FileSystemWatcher();
watcher.Path = directory;
watcher.Filter = FileName;
watcher.Changed += Watcher_Changed;
watcher.EnableRaisingEvents = true;
watcher.NotifyFilter = NotifyFilters.Attributes |
NotifyFilters.CreationTime |
NotifyFilters.DirectoryName |
NotifyFilters.FileName |
NotifyFilters.LastAccess |
NotifyFilters.LastWrite |
NotifyFilters.Security |
NotifyFilters.Size;
}
private void Watcher_Changed(object sender, FileSystemEventArgs e)
{
using (var fileStream = File.Open(Path.Combine(Directory, FileName), FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete | FileShare.Inheritable))
{
using (var reader = new StreamReader(fileStream))
{
string line;
while ((line = reader.ReadLine()) != null)
{
linesRead.Add(line);
}
}
}
}
}
}
right now TestMethodAfterClose succeeds and TestMethodAfterFlush fails. When I use the program BareTail and wait at the breakpoint, I see that it updates the display before the file is closed. So that gives me an indication that it's possible. I don't know if it's possible in C# and I might need to import some native functions using dllimport. The problem is I don't know where to look
How do I make both tests succeed, while not using a timer?
EDIT:
updated the FileWatcherTest class
Unfortunately Flush doesn't flush the thing you want. I find a lot articles to explain it, for example:
https://blogs.msdn.microsoft.com/alejacma/2011/03/23/filesystemwatcher-class-does-not-fire-change-events-when-notifyfilters-size-is-used/
There is a solution since .net 4, use another overload method of FileStream: Flush(bool)
var fs = writer.BaseStream as FileStream;
fs.Flush(true);
And you only give disk 10ms to react, maybe this is another problem.
After some searching, I discovered that the FileSystemWatcher only triggers an event after a file is closed as shown in this article. The article only mentions the date modified NotifyFilter, but in my testing I found that all Notifyfilters trigger after the file is closed and never while it's still open.
For that reason, it looks like tailing a file is only possible with a looped function that continuously monitors the file for extra rows. I used the code on this link as an example.
Here is my code working:
[TestClass]
public class FileWriteTests
{
[TestMethod]
public void TestMethodAfterClose_filetailing()
{
var currentDir = Environment.CurrentDirectory;
var fileToMonitor = "test.txt";
File.Delete(Path.Combine(currentDir, fileToMonitor));
List<string> output = new List<string>();
using (var watcherTest = new PersonalFileTail(currentDir, fileToMonitor))
{
watcherTest.StartTail(delegate (string line) { output.Add(line); });
using (var writer = new StreamWriter(Path.Combine(currentDir, fileToMonitor), true))
{
writer.WriteLine($"test");
writer.Flush();
}
System.Threading.Thread.Sleep(200);
watcherTest.StopTail();
}
System.Threading.Thread.Sleep(10);
Assert.AreEqual(1, output.Count);
Assert.AreEqual("test", output[0]);
}
[TestMethod]
public void TestMethodAfterFlush_filetailing()
{
// initiate file
var currentDir = Environment.CurrentDirectory;
var fileToMonitor = "test.txt";
File.Delete(Path.Combine(currentDir, fileToMonitor));
FileInfo info = new FileInfo(Path.Combine(currentDir, fileToMonitor));
List<string> output = new List<string>();
using (var watcherTest = new PersonalFileTail(currentDir, fileToMonitor))
{
watcherTest.StartTail(delegate (string line) { output.Add(line); });
using (var writer = new StreamWriter(Path.Combine(currentDir, fileToMonitor), true))
{
try
{
writer.WriteLine($"test");
writer.Flush();
System.Threading.Thread.Sleep(1000);
Assert.AreEqual(1, output.Count);
Assert.AreEqual("test", output[0]);
}
catch
{
Assert.Fail("Test failed");
}
}
watcherTest.StopTail();
}
}
public class PersonalFileTail : IDisposable
{
private string filename;
private string directory;
private Task fileTailTask;
private Action<string> handleResults;
private volatile bool runTask;
private long lastFilePosition;
public string FileName
{
get { return Path.Combine(directory, filename); }
}
public PersonalFileTail(string directory, string filename)
{
this.directory = directory;
this.filename = filename;
this.runTask = false;
lastFilePosition = 0;
}
public void StartTail(Action<string> handleResults)
{
this.handleResults = handleResults;
runTask = true;
fileTailTask = Task.Run(() => MonitorFileTask());
}
public void StopTail()
{
runTask = false;
fileTailTask.Wait();
}
public IEnumerable<string> ReadLinesFromFile()
{
using (StreamReader reader = new StreamReader(new FileStream(FileName,
FileMode.Open, FileAccess.Read, FileShare.ReadWrite)))
{
string line = "";
while ((line = reader.ReadLine()) != null)
{
yield return line;
}
lastFilePosition = reader.BaseStream.Length;
}
}
public void MonitorFileTask()
{
StreamReader reader = null;
FileStream stream = null;
try
{
using(stream = new FileStream(FileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
using (reader = new StreamReader(stream))
{
do
{
//if the file size has increased do something
if (reader.BaseStream.Length > lastFilePosition)
{
//seek to the last max offset
reader.BaseStream.Seek(lastFilePosition, SeekOrigin.Begin);
//read out of the file until the EOF
string line = "";
while ((line = reader.ReadLine()) != null)
{
handleResults(line);
}
//update the last max offset
lastFilePosition = reader.BaseStream.Position;
}
// sleep task for 100 ms
System.Threading.Thread.Sleep(100);
}
while (runTask);
}
}
catch
{
if (reader != null)
reader.Dispose();
if (stream != null)
stream.Dispose();
}
}
public void Dispose()
{
if(runTask)
{
runTask = false;
fileTailTask.Wait();
}
}
}
}
If anyone knows a way in which tailing can be done without using a timed function, I will accept that as the answer. Until that time, i feel that my answer is the only possible way to do this.
Related
I have a append-only log file which is monitored by a FileSystemWatcher. When the file is changed, Read() is called for the LogFile object.
The log file should be read line by line.
The goal is to read only changes i.e. lines add to the log file (skip already readed lines).
Thus the StreamReader should start to read from the position where it ended on the previous read.
My solution so far doesn't work. When I add
1
2
3
4
line by line in Notepad++ to my textfile & save each time when a line was added, the Debug output is
Initial read
1 //OK
2 //OK
3 //OK
1 //looks like the log file is read again from the beginning
2
3
4
Output should be
Initial read
1
2
3
4
Any ideas to solve this problem?
Console code
public class LogFile
{
public List<string> Contents { get; }
string _fullPath;
long position;
public LogFile(string fullPath)
{
if (File.Exists(fullPath))
{
_fullPath = fullPath;
Contents = new List<string>();
Read();
}
else
{
throw new FileNotFoundException($"{fullPath} not found");
}
}
public void Read(FileSystemWatcher fsw = null)
{
if (fsw != null)
fsw.EnableRaisingEvents = false; //set to false to prevent Changed event be fired twice
FileStream fs = null;
StreamReader sr = null;
try
{
fs = new FileStream(_fullPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
try
{
sr = new StreamReader(fs, Encoding.UTF8);
if (Contents.Count == 0)
{
Debug.WriteLine($"Initial read");
AddToContents(_fullPath, sr);
position = fs.Length; //store the length of the stream
}
else
{
sr.DiscardBufferedData();
sr.BaseStream.Seek(position, SeekOrigin.Begin);
AddToContents(_fullPath, sr);
position = fs.Length; //store the length of the stream
}
}
catch (Exception ex)
{
Debug.WriteLine($"Error while reading from {_fullPath}");
//log exception
}
finally
{
if (sr != null)
sr.Close();
}
}
catch (Exception ex)
{
Debug.WriteLine($"Error while opening {_fullPath}");
//log exception
}
finally
{
if (fs != null)
fs.Close();
if (fsw != null)
fsw.EnableRaisingEvents = true; //set raise events for the filesystemwatcher to true
}
}
private List<string> AddToContents(string fullPath, StreamReader sr)
{
List<string> newLines = new List<string>();
try
{
while (!sr.EndOfStream)
{
try
{
string line = sr.ReadLine();
if (line != string.Empty)
{
Contents.Add(line);
newLines.Add(line);
Debug.WriteLine(line);
}
}
catch (Exception ex)
{
Debug.WriteLine($"Error processing line in {fullPath}");
throw;
}
}
return newLines;
}
catch (Exception ex)
{
Debug.WriteLine($"Error while reading from {fullPath}");
throw;
}
}
}
class Program
{
static LogFile file;
static FileSystemWatcher fsw;
static void Main(string[] args)
{
string path = #"C:\Temp\test.txt";
file = new LogFile(path);
CreateWatcher(path);
Console.ReadKey();
}
private static FileSystemWatcher CreateWatcher(string fileNameFullPath)
{
fsw = new FileSystemWatcher(Path.GetDirectoryName(fileNameFullPath)); //constructor only accepts directory path
fsw.IncludeSubdirectories = false;
fsw.Filter = Path.GetFileName(fileNameFullPath); //filter for the given file
fsw.NotifyFilter = NotifyFilters.LastWrite;
fsw.EnableRaisingEvents = true;
fsw.Changed += Fsw_Changed;
return fsw;
}
private static void Fsw_Changed(object sender, FileSystemEventArgs e)
{
if (file != null)
file.Read(fsw);
}
}
Problem is
position = fs.Length; //store the length of the stream
You should store current position of the stream into position field not length of stream because sometimes FileStream.Length is zero (I don't know why)
this.position = fs.Position;
and check if FileStream.Length is zero skip that change
fs = new FileStream(this._fullPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
if (fs.Length != 0)
{
try
{
sr = new StreamReader(fs);
if (this.Contents.Count == 0)
{
......
Now it's working
I wrote program that listens on a directory (GUI - WPF). When the new file show up in this directory is sent to the printer. The problem occurs when I try to save a large file to this directory. I have to wait until the file is closed, and then send it to the printer. I have a function that checks if the file is open. But when I use it in the whole GUI hangs. How do I use this function asynchronously?
protected void newfile(object fscreated, FileSystemEventArgs Eventocc)
{
try
{
string CreatedFileName = Eventocc.Name;
FileInfo createdFile = new FileInfo(CreatedFileName);
string extension = createdFile.Extension;
string eventoccured = Eventocc.ChangeType.ToString();
fsLastRaised = DateTime.Now;
this.Dispatcher.Invoke((Action)(() =>
{
String file = "";
file = watchingFolder + "\\" + CreatedFileName;
//printing
this.Dispatcher.Invoke((Action)(() =>
{
FileInfo info = new FileInfo(file);
while (!IsFileReady(info)) { }
var t = new Thread(() => printFile(file, extension)); //send to printer
t.Start();
}));
}));
}
}
catch (Exception ex)
{
System.Windows.Forms.MessageBox.Show("Error");
}
}
IsFileReady function:
public static bool IsFileReady(FileInfo file)
{
FileStream stream = null;
try
{
stream = file.Open(FileMode.Open, FileAccess.Read, FileShare.None);
}
catch (IOException)
{
//the file is unavailable because it is:
//still being written to
//or being processed by another thread
//or does not exist (has already been processed)
return true;
}
finally
{
if (stream != null)
stream.Close();
}
//file is not locked
return false;
}
And printfile
public void printFile(string filepath, string ext)
{
ProcessStartInfo info = new ProcessStartInfo();
info.Verb = "print";
info.FileName = filepath;
info.CreateNoWindow = true;
info.WindowStyle = ProcessWindowStyle.Hidden;
Process p = new Process();
p.StartInfo = info;
p.Start();
p.WaitForInputIdle();
System.Threading.Thread.Sleep(3000);
if (false == p.CloseMainWindow())
p.Kill();
}
}
How can I correct this code to work with large files without hangs up?
EDIT:
For check new file I use FileSystemWatcher
private void start(object sender, RoutedEventArgs e)
{
if (watchingFolder == null)
{
}
else
{
fs = new FileSystemWatcher(watchingFolder, "*.*");
fs.EnableRaisingEvents = true;
fs.IncludeSubdirectories = true;
fs.Created += new FileSystemEventHandler(newfile);
btnSatrt.IsEnabled = false;
btnStop.IsEnabled = true;
}
}
You're executing while (!IsFileReady(info)) { } through Dispatcher.Invoke, that executes the code on the UI thread so it will block for sure the app.
You aren't interacting at all with the UI, so the correct approach is to execute it asynchronously, via Tasks and awaits or with a background thread via the ThreadPool and not using at all Dispatcher.Invoke.
Try to execute all code in the newfile event handler on a background thread by starting a new task:
protected async void newfile(object fscreated, FileSystemEventArgs Eventocc)
{
try
{
await Task.Run(() =>
{
string CreatedFileName = Eventocc.Name;
FileInfo createdFile = new FileInfo(CreatedFileName);
string extension = createdFile.Extension;
string eventoccured = Eventocc.ChangeType.ToString();
fsLastRaised = DateTime.Now;
string file = watchingFolder + "\\" + CreatedFileName;
FileInfo info = new FileInfo(file);
while (!IsFileReady(info)) { }
printFile(file, extension);
});
}
catch (Exception ex)
{
System.Windows.Forms.MessageBox.Show("Error");
}
}
Use a BackgroundWorker instead of Dispatcher.Invoke.
I am running into an issue with a FileSystemWatcher.
I listen to a certain text file and each new line (FileSystemWatcher.Changed) is handle.
Text file that I listen to it is on the local network and some computers can write to the file at the same time.
When a particular machine writing my text file I immediately take the last line in the file and handles it.
How can I verify if for example some machines write to the file at the same time that i took this string from each machine without miss ?
This is my class:
public class Watcher
{
private string _file;
public Watcher(string file)
{
_file = file;
}
public void startWatch()
{
FileSystemWatcher watcher = new FileSystemWatcher();
watcher.Path = Path.GetDirectoryName(_file);
watcher.Filter = Path.GetFileName(_file);
watcher.NotifyFilter = NotifyFilters.LastWrite;
watcher.Changed += watcher_Changed;
watcher.EnableRaisingEvents = true;
}
public void watcher_Changed(object sender, FileSystemEventArgs e)
{
readLastLine();
}
private void readLastLine()
{
string lastLine = string.Empty;
using (StreamReader sr = new StreamReader(File.Open(_file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)))
{
string str = sr.ReadToEnd();
int x = str.LastIndexOf('\n');
lastLine = str.Substring(x + 1);
}
validateString(lastLine);
}
private void validateString(string str)
{
string[] arr = str.Split(' ');
if (arr.Length != 2 && arr[0] != "start" && arr[0] != "stop" && arr[0] != "finish")
return;
// Handle the string...
}
}
I've got a windows form with save/loading of XML files and it asks the user where they want to save/load it. My problem is I dont know how to change this method to load the file from where the user wants and not where the streamreader specifies.
The code below is of my button and LoadValues Method.
private void Edittask_loadbuttonClick(
object sender, EventArgs e)
{
Stream myStream = null;
var sFile1 = new OpenFileDialog();
sFile1.InitialDirectory = "c:\\";
sFile1.Filter = "xml files (*.xml)|*.xml";
sFile1.FilterIndex = 2;
sFile1.RestoreDirectory = true;
if (sFile1.ShowDialog() == DialogResult.OK)
{
try
{
if ((myStream = sFile1.OpenFile()) != null)
{
using (myStream)
{
var v = LoadValues();
this.load_task1_name.Text =
v.task1_name;
this.load_task1_desc.Text =
v.task1_desc;
this.load_task1_date.Value =
v.task1_date;
this.load_checkbox.Checked =
v.task1_checkbox;
}
}
}
catch (Exception ex)
{
MessageBox.Show(
"Error: Could not read file from disk. Original error: " +
ex.Message);
}
}
}
public Values LoadValues()
{
var serializer = new XmlSerializer(typeof (Values));
using (
TextReader textReader = new StreamReader(
"E:\\hello.xml")
)
{
return
(Values) serializer.Deserialize(textReader);
}
}
I would pass the Stream from the OpenFileDialog to LoadValues(...), and use that to construct your StreamReader:
public Values LoadValues(Stream stream)
{
XmlSerializer serializer = new XmlSerializer(typeof(Values));
using (TextReader textReader = new StreamReader(stream))
{
return (Values)serializer.Deserialize(textReader);
}
}
and
if ((myStream = sFile1.OpenFile()) != null)
{
using (myStream)
{
Values v = LoadValues(myStream);
...
}
}
You need to pass the Stream as a parameter to your function.
I have a slight problem. What my application is supose to do, is to watch a folder for any newly copied file with the extention '.XSD' open the file and assign the lines to an array. After that the data from the array should be inserted into a MySQL database, then move the used file to another folder if it's done.
The problem is that the application works fine with the first file, but as soon as the next file is copied to the folder I get this exception for example: 'The process cannot access the file 'C:\inetpub\admission\file2.XPD' because it is being used by another process'.
If two files on the onther hand is copied at the same time there's no problem at all.
The following code is on the main window:
public partial class Form1 : Form
{
static string folder = specified path;
static FileProcessor processor;
public Form1()
{
InitializeComponent();
processor = new FileProcessor();
InitializeWatcher();
}
static FileSystemWatcher watcher;
static void InitializeWatcher()
{
watcher = new FileSystemWatcher();
watcher.Path = folder;
watcher.Created += new FileSystemEventHandler(watcher_Created);
watcher.EnableRaisingEvents = true;
watcher.Filter = "*.XPD";
}
static void watcher_Created(object sender, FileSystemEventArgs e)
{
processor.QueueInput(e.FullPath);
}
}
As you can see the file's path is entered into a queue for processing which is on another class called FileProcessor:
class FileProcessor
{
private Queue<string> workQueue;
private Thread workerThread;
private EventWaitHandle waitHandle;
public FileProcessor()
{
workQueue = new Queue<string>();
waitHandle = new AutoResetEvent(true);
}
public void QueueInput(string filepath)
{
workQueue.Enqueue(filepath);
if (workerThread == null)
{
workerThread = new Thread(new ThreadStart(Work));
workerThread.Start();
}
else if (workerThread.ThreadState == ThreadState.WaitSleepJoin)
{
waitHandle.Set();
}
}
private void Work()
{
while (true)
{
string filepath = RetrieveFile();
if (filepath != null)
ProcessFile(filepath);
else
waitHandle.WaitOne();
}
}
private string RetrieveFile()
{
if (workQueue.Count > 0)
return workQueue.Dequeue();
else
return null;
}
private void ProcessFile(string filepath)
{
string xName = Path.GetFileName(filepath);
string fName = Path.GetFileNameWithoutExtension(filepath);
string gfolder = specified path;
bool fileInUse = true;
string line;
string[] itemArray = null;
int i = 0;
#region Declare Db variables
//variables for each field of the database is created here
#endregion
#region Populate array
while (fileInUse == true)
{
FileStream fs = new FileStream(filepath, FileMode.Open, FileAccess.Read,
FileShare.ReadWrite);
StreamReader reader = new StreamReader(fs);
itemArray = new string[75];
while (!reader.EndOfStream == true)
{
line = reader.ReadLine();
itemArray[i] = line;
i++;
}
fs.Flush();
reader.Close();
reader.Dispose();
i = 0;
fileInUse = false;
}
#endregion
#region Assign Db variables
//here all the variables get there values from the array
#endregion
#region MySql Connection
//here the connection to mysql is made and the variables are inserted into the db
#endregion
#region Test and Move file
if (System.IO.File.Exists(gfolder + xName))
{
System.IO.File.Delete(gfolder + xName);
}
Directory.Move(filepath, gfolder + xName);
#endregion
}
}
The problem I get occurs in the Populate array region. I read alot of other threads and was lead to believe that by flushing the file stream would help...
I am also thinking of adding a try..catch for if the file process was successful, the file is moved to gfolder and if it failed, moved to bfolder
Any help would be awesome
Tx
You're not disposing of your FileStream instance, so a lock remains on the file. Change your code to use using blocks:
using (var fileStream = new FileStream(...))
{
using (var reader = new StreamReader(fileStream))
{
}
}
These using blocks will ensure the instances are correctly disposed of.
Also, why are you calling Flush on the file stream? You're not writing anything with it...
I would suggest :
1° use the using syntax on StreamReader
2° use the using syntax on FileStream