I have to implement a Delay Queue in C# as there is no standard implementation of delay queue present in the C#. I am looking to use System.Threading.Timer for implementing delayed enqueue of a node.
public class DelayQueue<T>
{
private Queue queue<T> = new Queue<T>();
public void Enqueue(Object object)
{
this.queue.Enqueue(object as T);
}
public void Enqueue(T node, TimeSpan dueTime)
{
new System.Threading.Timer(this.Enqueue, node, dueTime, -1);
}
.
.
.
}
This approach looks fine to me but since I am new to C#(from C background), I want someones opinion that whether it is the right way or are there any better and more effective methods of doing the same?
I don't think that creating timers for each item is a good idea. Anyway, you only need to get first ready item when you dequeue item from queue, then you just can store time when item will be ready:
public class DelayQueue<T>
{
private List<DelayQueueItem<T>> items = new List<DelayQueueItem<T>>();
public void Enqueue(T item)
{
Enqueue(item, TimeSpan.Zero);
}
public void Enqueue(T item, TimeSpan delay)
{
items.Add(new DelayQueueItem<T>()
{
Value = item,
ReadyTime = DateTime.Now.Add(delay)
});
}
public T Dequeue()
{
DateTime now = DateTime.Now;
var item = items.FirstOrDefault(i => i.ReadyTime <= now);
if (item != null)
{
items.Remove(item);
return item.Value;
}
return default(T);
}
private class DelayQueueItem<T>
{
public T Value { get; set; }
public DateTime ReadyTime { get; set; }
}
}
UPDATE (blocking queue with waiting timeout)
public T Dequeue()
{
return Dequeue(TimeSpan.Zero);
}
public T Dequeue(TimeSpan timeout)
{
DateTime startTime = DateTime.Now;
do
{
DateTime now = DateTime.Now;
var item = items.FirstOrDefault(i => i.ReadyTime <= now);
if (item == null)
continue;
items.Remove(item);
return item.Value;
}
while (DateTime.Now - startTime < timeout);
return default(T);
}
Usage:
DelayQueue<string> queue = new DelayQueue<string>();
queue.Enqueue("world", new TimeSpan(0, 0, 1));
queue.Enqueue("hello");
queue.Enqueue(",");
TimeSpan timeout = new TimeSpan(0, 0, 2);
Console.WriteLine(queue.Dequeue());
Console.WriteLine(queue.Dequeue(timeout));
Console.WriteLine(queue.Dequeue(timeout));
Related
I doing a small project to map a network (routers only) using SNMP. In order to speed things up, I´m trying to have a pool of threads responsible for doing the jobs I need, apart from the first job which is done by the main thread.
At this time I have two jobs, one takes a parameter the other doesn´t:
UpdateDeviceInfo(NetworkDevice nd)
UpdateLinks() *not defined yet
What I´m trying to achieve is to have those working threads waiting for a job to
appear on a Queue<Action> and wait while it is empty. The main thread will add the first job and then wait for all workers, which might add more jobs, to finish before starting adding the second job and wake up the sleeping threads.
My problem/questions are:
How to define the Queue<Actions> so that I can insert the methods and the parameters if any. If not possible I could make all functions accept the same parameter.
How to launch the working threads indefinitely. I not sure where should I create the for(;;).
This is my code so far:
public enum DatabaseState
{
Empty = 0,
Learning = 1,
Updating = 2,
Stable = 3,
Exiting = 4
};
public class NetworkDB
{
public Dictionary<string, NetworkDevice> database;
private Queue<Action<NetworkDevice>> jobs;
private string _community;
private string _ipaddress;
private Object _statelock = new Object();
private DatabaseState _state = DatabaseState.Empty;
private readonly int workers = 4;
private Object _threadswaitinglock = new Object();
private int _threadswaiting = 0;
public Dictionary<string, NetworkDevice> Database { get => database; set => database = value; }
public NetworkDB(string community, string ipaddress)
{
_community = community;
_ipaddress = ipaddress;
database = new Dictionary<string, NetworkDevice>();
jobs = new Queue<Action<NetworkDevice>>();
}
public void Start()
{
NetworkDevice nd = SNMP.GetDeviceInfo(new IpAddress(_ipaddress), _community);
if (nd.Status > NetworkDeviceStatus.Unknown)
{
database.Add(nd.Id, nd);
_state = DatabaseState.Learning;
nd.Update(this); // The first job is done by the main thread
for (int i = 0; i < workers; i++)
{
Thread t = new Thread(JobRemove);
t.Start();
}
lock (_statelock)
{
if (_state == DatabaseState.Learning)
{
Monitor.Wait(_statelock);
}
}
lock (_statelock)
{
if (_state == DatabaseState.Updating)
{
Monitor.Wait(_statelock);
}
}
foreach (KeyValuePair<string, NetworkDevice> n in database)
{
using (System.IO.StreamWriter file = new System.IO.StreamWriter(n.Value.Name + ".txt")
{
file.WriteLine(n);
}
}
}
}
public void JobInsert(Action<NetworkDevice> func, NetworkDevice nd)
{
lock (jobs)
{
jobs.Enqueue(item);
if (jobs.Count == 1)
{
// wake up any blocked dequeue
Monitor.Pulse(jobs);
}
}
}
public void JobRemove()
{
Action<NetworkDevice> item;
lock (jobs)
{
while (jobs.Count == 0)
{
lock (_threadswaitinglock)
{
_threadswaiting += 1;
if (_threadswaiting == workers)
Monitor.Pulse(_statelock);
}
Monitor.Wait(jobs);
}
lock (_threadswaitinglock)
{
_threadswaiting -= 1;
}
item = jobs.Dequeue();
item.Invoke();
}
}
public bool NetworkDeviceExists(NetworkDevice nd)
{
try
{
Monitor.Enter(database);
if (database.ContainsKey(nd.Id))
{
return true;
}
else
{
database.Add(nd.Id, nd);
Action<NetworkDevice> action = new Action<NetworkDevice>(UpdateDeviceInfo);
jobs.Enqueue(action);
return false;
}
}
finally
{
Monitor.Exit(database);
}
}
//Job1 - Learning -> Update device info
public void UpdateDeviceInfo(NetworkDevice nd)
{
nd.Update(this);
try
{
Monitor.Enter(database);
nd.Status = NetworkDeviceStatus.Self;
}
finally
{
Monitor.Exit(database);
}
}
//Job2 - Updating -> After Learning, create links between neighbours
private void UpdateLinks()
{
}
}
Your best bet seems like using a BlockingCollection instead of the Queue class. They behave effectively the same in terms of FIFO, but a BlockingCollection will let each of your threads block until an item can be taken by calling GetConsumingEnumerable or Take. Here is a complete example.
http://mikehadlow.blogspot.com/2012/11/using-blockingcollection-to-communicate.html?m=1
As for including the parameters, it seems like you could use closure to enclose the NetworkDevice itself and then just enqueue Action instead of Action<>
My first attempt to define a measurement schedule was:
var schedule = Observable.Concat(
Observable.Interval(TimeSpan.FromSeconds(1)).Take(3),
Observable.Interval(TimeSpan.FromSeconds(3)).Take(3),
Observable.Interval(TimeSpan.FromSeconds(5)));
Unfortunately, it restarts when unsubscribed and resubscribed, which is not a desired behavior in my case. Therefore, I came with something like this:
class Schedule : IObservable<DateTime>, IDisposable
{
readonly ISubject<DateTime> _subject;
readonly IDisposable _subscrption;
public Schedule()
{
_subject = new BehaviorSubject<DateTime>(DateTime.UtcNow);
_subscrption = Observable.Concat(
Observable.Interval(TimeSpan.FromSeconds(1)).Take(3),
Observable.Interval(TimeSpan.FromSeconds(3)).Take(3),
Observable.Interval(TimeSpan.FromSeconds(5)))
.Select(i => DateTime.UtcNow)
.Subscribe(_subject);
}
public IDisposable Subscribe(IObserver<DateTime> observer)
{
return _subject.Subscribe(observer);
}
public void Dispose()
{
_subscrption.Dispose();
}
}
It works but requires disposing after use. Are there any simple way to define Schedule without exposing IDisposable?
If you will be unsubscribing all of your observers and then resubscribing, there really is no way around keeping a Disposable around for tracking the connection. Since your Observable would not have anyway of directly knowing if the latest unsubscribe is really the last one.
I would recommend the use of Generate instead of Concatenating Observables
IConnectableObservable<DateTime> source = Observable.Generate(
0,
_ => true,
x => x + 1,
x => DateTime.UtcNow,
x => {
if (x < 3) return TimeSpan.FromSeconds(1);
else if (x < 5) return TimeSpan.FromSeconds(3);
else return TimeSpan.FromSeconds(5);
}).Publish();
Publish will turn your Observable into a ConnectableObservable, and gives you two options for managing when the source actually produces events.
//1) Explicit connect
IDisposable connection = source.Connect();
//2) RefCounted connection
IObservable<DateTime> rcSource = source.RefCount();
In the first version the source will become "hot" once you connect, and when you want to disconnect you should dispose of connection.
In the second the source will automatically disconnect when it has no more observers.
see also hot vs cold observables
Here is the solution I defined which does not require disposing. It is pretty complex though, so I would highly appreciate if something can be simplified using Rx instrumentation.
The usage:
IObservable<DateTime> schedule = new Schedule()
.Setup(TimeSpan.FromSeconds(1), 3)
.Setup(TimeSpan.FromSeconds(3), 3)
.Setup(TimeSpan.FromSeconds(5));
Where Schedule is:
class Schedule : IObservable<DateTime>
{
readonly TimeSpan TimerPrecision = TimeSpan.FromMilliseconds(1);
readonly IEnumerable<TimeSpan> Intervals;
readonly IEnumerator<DateTime> Event;
public Schedule()
: this(new TimeSpan[0])
{
}
Schedule(IEnumerable<TimeSpan> intervals)
{
Intervals = intervals;
Event = Start();
Event.MoveNext();
}
public Schedule Setup(TimeSpan interval)
{
return Setup(interval, Int32.MaxValue);
}
public Schedule Setup(TimeSpan interval, int repeat)
{
return new Schedule(
Intervals.Concat(
Enumerable.Repeat(interval, repeat)));
}
public IDisposable Subscribe(IObserver<DateTime> observer)
{
var timer = new System.Timers.Timer() { AutoReset = true };
timer.Elapsed += (s, a) =>
{
observer.OnNext(DateTime.UtcNow);
if (!TryArm(timer))
observer.OnCompleted();
};
if (!TryArm(timer))
observer.OnCompleted();
return timer;
}
IEnumerator<DateTime> Start()
{
var time = DateTime.UtcNow;
yield return time;
foreach (var interval in Intervals)
{
time += interval;
yield return time;
}
}
TimeSpan Delay()
{
var now = DateTime.UtcNow;
lock (Event)
while (Event.Current - DateTime.UtcNow < TimerPrecision)
Event.MoveNext();
return Event.Current - now;
}
bool TryArm(System.Timers.Timer timer)
{
try
{
timer.Interval = Delay().TotalMilliseconds;
timer.Start();
return true;
}
catch(ObjectDisposedException)
{
return false;
}
catch(InvalidOperationException)
{
return false;
}
}
}
Update - solved
The final solution differs a bit from Brandon's suggestion but his answer brought me on the right track.
class State
{
public int Offset { get; set; }
public HashSet<string> UniqueImageUrls = new HashSet<string>();
}
public IObservable<TPicture> GetPictures(ref object _state)
{
var localState = (State) _state ?? new State();
_state = localState;
return Observable.Defer(()=>
{
return Observable.Defer(() => Observable.Return(GetPage(localState.Offset)))
.SubscribeOn(TaskPoolScheduler.Default)
.Do(x=> localState.Offset += 20)
.Repeat()
.TakeWhile(x=> x.Count > 0)
.SelectMany(x=> x)
.Where(x=> !localState.UniqueImageUrls.Contains(x.ImageUrl))
.Do(x=> localState.UniqueImageUrls.Add(x.ImageUrl));
});
}
IList<TPicture> GetPage(int offset)
{
...
return result;
}
Original Question
I'm currently struggling with the following problem. The PictureProvider implementation shown below is working with an offset variable used for paging results of a backend service providing the actual data. What I would like to implement is an elegant solution making the current offset available to the consumer of the observable to allow for resuming the observable sequence at a later time at the correct offset. Resuming is already accounted for by the intialState argument to GetPictures().
Recommendations for improving the code in a more RX like fashion would be welcome as well. I'm actually not so sure if the Task.Run() stuff is appropriate here.
public class PictureProvider :
IPictureProvider<Picture>
{
#region IPictureProvider implementation
public IObservable<Picture> GetPictures(object initialState)
{
return Observable.Create<Picture>((IObserver<Picture> observer) =>
{
var state = new ProducerState(initialState);
ProducePictures(observer, state);
return state;
});
}
#endregion
void ProducePictures(IObserver<Picture> observer, ProducerState state)
{
Task.Run(() =>
{
try
{
while(!state.Terminate.WaitOne(0))
{
var page = GetPage(state.Offset);
if(page.Count == 0)
{
observer.OnCompleted();
break;
}
else
{
foreach(var picture in page)
observer.OnNext(picture);
state.Offset += page.Count;
}
}
}
catch (Exception ex)
{
observer.OnError(ex);
}
state.TerminateAck.Set();
});
}
IList<Picture> GetPage(int offset)
{
var result = new List<Picture>();
... boring web service call here
return result;
}
public class ProducerState :
IDisposable
{
public ProducerState(object initialState)
{
Terminate = new ManualResetEvent(false);
TerminateAck = new ManualResetEvent(false);
if(initialState != null)
Offset = (int) initialState;
}
public ManualResetEvent Terminate { get; private set; }
public ManualResetEvent TerminateAck { get; private set; }
public int Offset { get; set; }
#region IDisposable implementation
public void Dispose()
{
Terminate.Set();
TerminateAck.WaitOne();
Terminate.Dispose();
TerminateAck.Dispose();
}
#endregion
}
}
I suggest refactoring your interface to yield the state as part of the data. Now the client has what they need to resubscribe where they left off.
Also, once you start using Rx, you should find that using synchronization primitives like ManualResetEvent are rarely necessary. If you refactor your code so that retrieving each page is its own Task, then you can eliminate all of that synchronization code.
Also, if you are calling a "boring web service" in GetPage, then just make it async. This gets rid of the need to call Task.Run among other benefits.
Here is a refactored version, using .NET 4.5 async/await syntax. It could also be done without async/await. I also added a GetPageAsync method that uses Observable.Run just in case you really cannot convert your webservice call to be asynchronous
/// <summary>A set of pictures</summary>
public struct PictureSet
{
public int Offset { get; private set; }
public IList<Picture> Pictures { get; private set; }
/// <summary>Clients will use this property if they want to pick up where they left off</summary>
public int NextOffset { get { return Offset + Pictures.Count; } }
public PictureSet(int offset, IList<Picture> pictures)
:this() { Offset = offset; Pictures = pictures; }
}
public class PictureProvider : IPictureProvider<PictureSet>
{
public IObservable<PictureSet> GetPictures(int offset = 0)
{
// use Defer() so we can capture a copy of offset
// for each observer that subscribes (so multiple
// observers do not update each other's offset
return Observable.Defer<PictureSet>(() =>
{
var localOffset = offset;
// Use Defer so we re-execute GetPageAsync()
// each time through the loop.
// Update localOffset after each GetPageAsync()
// completes so that the next call to GetPageAsync()
// uses the next offset
return Observable.Defer(() => GetPageAsync(localOffset))
.Select(pictures =>
{
var s = new PictureSet(localOffset, pictures);
localOffset += pictures.Count;
})
.Repeat()
.TakeWhile(pictureSet => pictureSet.Pictures.Count > 0);
});
}
private async Task<IList<Picture>> GetPageAsync(int offset)
{
var data = await BoringWebServiceCallAsync(offset);
result = data.Pictures.ToList();
}
// this version uses Observable.Run() (which just uses Task.Run under the hood)
// in case you cannot convert your
// web service call to be asynchronous
private IObservable<IList<Picture>> GetPageAsync(int offset)
{
return Observable.Run(() =>
{
var result = new List<Picture>();
... boring web service call here
return result;
});
}
}
Clients just need to add a SelectMany call to get their IObservable<Picture>. They can choose to store the pictureSet.NextOffset if they wish.
pictureProvider
.GetPictures()
.SelectMany(pictureSet => pictureSet.Pictures)
.Subscribe(picture => whatever);
Instead of thinking about how to save the subscription state, I would think about how to replay the state of the inputs (i.e. I'd try to create a serializable ReplaySubject that, on resume, would just resubscribe and catch back up to the current state).
I have a single observable sequence of messages. There is a set of subscribers that can handle these messages. Each subscriber has an execution priority. Each message must be handled once by the highest priority subscriber chosen from the list of currently subscribed subscribers. Subscribers are constantly subscribed/unsubscribed from the sequence, so we don't know the number and priorities of subscribers when constructing the sequence. Is it a possible/viable solution using rx?
To illustrate:
public class Message
{
public string Value { get; set; }
public bool IsConsumed { get; set; }
}
var subject = new Subject<Message>();
var sequence = subject.Publish().RefCount();
Action<Message, int> subscriber = (m, priority) =>
{
if (!m.IsConsumed)
{
m.IsConsumed = true;
Trace.WriteLine(priority);
}
};
var s2 = sequence.Priority(2).Subscribe(m => subscriber(m, 2));
var s1 = sequence.Priority(1).Subscribe(m => subscriber(m, 1));
subject.OnNext(new Message()); // output: 1
s1.Dispose();
subject.OnNext(new Message()); // output: 2
The missing piece to make this solution work is the Priority method which do not exist in Rx library.
This was a very interesting problem...
So, first off: I am not aware of any intrinsic Rx operators that can achieve a "routing" effect similar to what you want in this Priority extension.
That said, I was playing around in LINQPad over lunch today, and came up with a (very) hacky proof of concept that appears to work:
First, your message class
public class Message
{
public string Value { get; set; }
public bool IsConsumed { get; set; }
}
Next, the extension method wrapper-class:
public static class Ext
{
public static PrioritizedObservable<T> Prioritize<T>(this IObservable<T> source)
{
return new PrioritizedObservable<T>(source);
}
}
And what is this PrioritizedObservable<T>?
public class PrioritizedObservable<T>
: IObservable<T>, IObserver<T>, IDisposable
{
private IObservable<T> _source;
private ISubject<T,T> _intermediary;
private IList<Tuple<int, Subject<T>>> _router;
public PrioritizedObservable(IObservable<T> source)
{
// Make sure we don't accidentally duplicate subscriptions
// to the underlying source
_source = source.Publish().RefCount();
// A proxy from the source to our internal router
_intermediary = Subject.Create(this, _source);
_source.Subscribe(_intermediary);
// Holds per-priority subjects
_router = new List<Tuple<int, Subject<T>>>();
}
public void Dispose()
{
_intermediary = null;
foreach(var entry in _router)
{
entry.Item2.Dispose();
}
_router.Clear();
}
private ISubject<T,T> GetFirstListener()
{
// Fetch the first subject in our router
// ordered by priority
return _router.OrderBy(tup => tup.Item1)
.Select(tup => tup.Item2)
.FirstOrDefault();
}
void IObserver<T>.OnNext(T value)
{
// pass along value to first in line
var nextListener = GetFirstListener();
if(nextListener != null)
nextListener.OnNext(value);
}
void IObserver<T>.OnError(Exception error)
{
// pass along error to first in line
var nextListener = GetFirstListener();
if(nextListener != null)
nextListener.OnError(error);
}
void IObserver<T>.OnCompleted()
{
var nextListener = GetFirstListener();
if(nextListener != null)
nextListener.OnCompleted();
}
public IDisposable Subscribe(IObserver<T> obs)
{
return PrioritySubscribe(1, obs);
}
public IDisposable PrioritySubscribe(int priority, IObserver<T> obs)
{
var sub = new Subject<T>();
var subscriber = sub.Subscribe(obs);
var entry = Tuple.Create(priority, sub);
_router.Add(entry);
_intermediary.Subscribe(sub);
return Disposable.Create(() =>
{
subscriber.Dispose();
_router.Remove(entry);
});
}
}
And a test harness:
void Main()
{
var subject = new Subject<Message>();
var sequence = subject.Publish().RefCount().Prioritize();
Action<Message, int> subscriber = (m, priority) =>
{
if (!m.IsConsumed)
{
m.IsConsumed = true;
Console.WriteLine(priority);
}
};
var s3 = sequence.PrioritySubscribe(3, Observer.Create<Message>(m => subscriber(m, 3)));
var s2 = sequence.PrioritySubscribe(2, Observer.Create<Message>(m => subscriber(m, 2)));
var s1 = sequence.PrioritySubscribe(1, Observer.Create<Message>(m => subscriber(m, 1)));
var s11 = sequence.PrioritySubscribe(1, Observer.Create<Message>(m => subscriber(m, 1)));
subject.OnNext(new Message()); // output: 1
s1.Dispose();
subject.OnNext(new Message()); // output: 1
s11.Dispose();
subject.OnNext(new Message()); // output: 2
s2.Dispose();
subject.OnNext(new Message()); // output: 3
sequence.Dispose();
}
my program should run maximum (N) job at a time. if there is more job needs to be run it is store in temp storage and after completing one of the currently running job then i'll pick another trigger base on how much the trigger fails to start and it's priority, and then fire its job
at initialization phase, I create for example 5 job with 5 corresponding trigger and add it to scheduler everything's fine until second job is running but TriggerComplete of the trigger listener is not firing for picking up another job to run, could you please tell me where im wrong ??
public class CrawlerTriggerListener : ITriggerListener
{
private int maxConcurrentCrawling = 1;
private int currentCount = 0;
private object syncLock = new object();
private Dictionary fireDic = new Dictionary();
public string Name { get { return "listener"; } }
public void TriggerFired(Trigger trigger, JobExecutionContext context)
{
if (fireDic.Count == 0)
{
IScheduler sched = context.Scheduler;
string[] triggerNameList = sched.GetTriggerNames("triggerGroup");
foreach (string triggerName in triggerNameList)
{
MissfiredInfo missedInfo = new MissfiredInfo();
missedInfo.TriggerName = triggerName;
missedInfo.Priority = sched.GetTrigger(triggerName, "triggerGroup").Priority;
fireDic.Add(triggerName, missedInfo);
}
}
}
public bool VetoJobExecution(Trigger trigger, JobExecutionContext context)
{
lock (syncLock)
{
if (currentCount < maxConcurrentCrawling)
{
currentCount++;
fireDic[trigger.Name].FailCount = 0;
fireDic[trigger.Name].LastFireTime = DateTime.UtcNow;
return false;
}
else
{
fireDic[trigger.Name].LastFireTime = DateTime.UtcNow;
fireDic[trigger.Name].FailCount++;
return true;
}
}
}
public void TriggerMisfired(Trigger trigger) { }
public void TriggerComplete(Trigger trigger, JobExecutionContext context, SchedulerInstruction triggerInstructionCode)
{
lock (syncLock)
{
currentCount--;
var validCandidate = new Dictionary<string, int>();
foreach (KeyValuePair<string, MissfiredInfo> fireDicItem in fireDic)
if (fireDicItem.Value.FailCount > 0)
validCandidate.Add(fireDicItem.Key, fireDicItem.Value.FailCount * 73 + fireDicItem.Value.Priority);
if (validCandidate.Count > 0)
{
var sorted = (from entry in validCandidate orderby entry.Value ascending select entry);
string triggerName = sorted.First().Key;
fireDic[triggerName].LastFireTime = DateTime.UtcNow;
fireDic[triggerName].FailCount = 0;
string jobName = context.Scheduler.GetTrigger(triggerName, "triggerGroup").JobName;
currentCount++;
context.Scheduler.TriggerJob(jobName, "jobGroup");
}
}
}
}
Okay, so again, I'm not sure where you are instantiating the TriggerListener, but you might want to verify that you are adding the TriggerListener to the Scheduler.
http://quartznet.sourceforge.net/tutorial/lesson_7.html
See that the scheduler instance has a method for "adding" (or registering) listeners. If you don't do that, the events will never fire.