I have following problem: Service is being run on the server. It has timer that ticks every 5 seconds. It is being modeled to produce items for a consumer. Items are being consumed in multi thread.
I've added logging but I cannot find out when and where any problem occurs and this just blocks. No exception or error. My goal is to keep getting requests to be processed from db and consume them. Timer is being a producer.
public class CustomProducerConsumer<T> : IDisposable
{
private readonly BlockingCollection<T> blockingCollection;
private readonly Action<T> consumeItem;
private readonly Task[] workers;
public CustomProducerConsumer(Action<T> consumeItem,
int degreeOfParallelism,
int capacity = 1024)
{
this.consumeItem = consumeItem;
this.blockingCollection = new BlockingCollection<T>(capacity);
this.workers = Enumerable.Range(1, degreeOfParallelism)
.Select(_ => Task.Factory.StartNew(Worker,
TaskCreationOptions.LongRunning))
.ToArray();
}
public void Dispose()
{
// Unblock all workers even if the client
// didn't call CompleteProcessing
if (!this.blockingCollection.IsAddingCompleted)
{
this.blockingCollection.CompleteAdding();
}
Task.WaitAll(this.workers);
this.blockingCollection.Dispose();
}
public void Process(T item)
{
this.blockingCollection.TryAdd(item);
}
private void Worker()
{
foreach (var item in this.blockingCollection.GetConsumingEnumerable())
{
this.consumeItem(item);
}
}
}
Here is my code from service:
private readonly BlockingCollection<StitchingRequestProcessingModel> requestsToBeProcessed =
new BlockingCollection<StitchingRequestProcessingModel>(10);
private readonly BlockingCollection<Dictionary<int, StitchingRequest[]>> pendingRequests =
new BlockingCollection<Dictionary<int, StitchingRequest[]>>(10);
private readonly Timer timer;
public Service()
{
InitializeComponent();
this.produceConsumer =
new CustomProducerConsumer<StitchingRequestModel>(this.ProcessItems,
Environment.ProcessorCount);
this.timer = new Timer(o =>
{
this.TimerElapsed();
this.timer.Change(TimeSpan.FromSeconds(5), Timeout.InfiniteTimeSpan);
}, null, TimeSpan.Zero, Timeout.InfiniteTimeSpan);
this.ConsumeRequests();
}
public void TimerElapsed()
{
try
{
//this just adds into the list an item to ping the db for pending requests when available
this.requestsToBeProcessed.Add(new StitchingRequestProcessingModel());
this.pendingRequests.TryTake(out Dictionary<int, Request[]> requests);
if (requests == null)
{
return;
}
foreach (KeyValuePair<int, Request[]> request in requests)
{
this.produceConsumer.Process(new StitchingRequestModel(request));
}
}
catch (Exception exception)
{
this.errorLogger.Error(exception.Message);
}
}
private void ConsumeRequests()
{
Task.Factory.StartNew(() =>
{
while (!this.requestsToBeProcessed.IsCompleted)
{
if (this.tokenSource.Token.IsCancellationRequested)
{
break;
}
StitchingRequestProcessingModel data = null;
try
{
data = this.requestsToBeProcessed.Take();
}
catch (InvalidOperationException)
{
}
if (data == null)
{
continue;
}
try
{
// this just executes sql query to get those request from db
var requests = this.requestService.GetPendingRequests();
this.pendingRequests.Add(requests);
}
catch (Exception exception)
{
this.errorLogger.Error(exception.Message, "Failed to get pending requests");
}
}
},
this.tokenSource.Token,
TaskCreationOptions.LongRunning, TaskScheduler.Current);
}
private void ProcessItems(StitchingRequestModel model)
{
foreach (StitchingRequest request in model.Requests)
{
this.requestsToBeProcessed.Add(new StitchingRequestProcessingModel(request);
}
}
Main reason why I placed consuming items into blocking collection is Nhibernate. It is giving me issues while doing multithreading. No ideas what else to try, nor why this approach is not working. I do not want to call CompleteAdding on blocking collection since I need requests to be added and just processed in first available thread.
Timer on each elapsed event will try to create an a pending request that will be added into blocking collection and processed on first available turn. Service runs for 2~3h and just stops. ProcessItems method can be long running. CPU is 8 core.
UPDATE
Added cancelation for consumer task.
Problem was solved with not working with entity object between consumer and producer. Created dto for info needed.
Related
I am working with Azure Event Hub producer client and reading messages off of a kafka stream then pass it along to deserialize/map, then pass to Event Hub. I have the consume loop which is creating a task for each consume and then two methods to do processing(this seems to have greatly improved the speed from the kafka lag perspective. However, Event hub makes you create an event batch which I don't necessarily want to use. I just want to send the data one message at a time for now. In order to create a new batch I have to call Dispose(). I am running into an issue where there's another call to the function by the the time I call Dispose() and I get an error saying the object is being used by event hub.
I've also tried using the overload for eventHubProducerClient.SendAsync that allows you to pass in a IEnumerable but i'm running into the same issue with that.
So I believe this to be a synchronization issue, or maybe I need to do a lock somewhere?
Any help would be appreciated.
public void Execute()
{
using (_consumer)
{
try
{
_consumer.Subscribe(_streamConsumerSettings.Topic);
while (true)
{
var result = _consumer.Consume(1000);
if (result == null)
{
continue;
}
var process = Task.Factory.StartNew(() => ProcessMessage(result?.Message?.Value));
var send = process.ContinueWith(t => SendMessage(process.Result));
}
}
catch (ConsumeException e)
{
_logger.LogError(e, e.StackTrace ?? e.Message);
_cancelConsume = true;
_consumer.Close();
RestartConsumer();
}
}
}
public static EquipmentJson ProcessMessage(byte[] result)
{
var json = _messageProcessor.DeserializeAndMap(result);
return json;
}
public static void SendMessage(EquipmentJson message)
{
try
{
_eventHubClient.AddToBatch(message);
}
catch (Exception e)
{
_logger.LogError(e, e.StackTrace ?? e.Message);
}
}
public async Task AddToBatch(EquipmentJson message)
{
if
(!string.IsNullOrEmpty(message.EquipmentLocation))
{
try
{
var batch = await _equipmentLocClient.CreateBatchAsync();
batch.TryAdd(new EventData(Encoding.UTF8.GetBytes(message.EquipmentLocation)));
await _eventHubProducerClient.SendAsync(batch);
batch.Dispose();
_logger.LogInformation($"Data sent {DateTimeOffset.UtcNow}");
}
catch (Exception e)
{
_logger.LogError(e, e.StackTrace ?? e.Message);
}
}
}
public class EventHubClient : IEventHubClient
{
private readonly ILoggerAdapter<EventHubClient> _logger;
private readonly EventHubClientSettings _eventHubClientSettings;
private IMapper _mapper;
private static EventHubProducerClient _equipmentLocClient;
public EventHubClient(ILoggerAdapter<EventHubClient> logger, EventHubClientSettings eventHubClientSettings, IMapper mapper)
{
_logger = logger;
_eventHubClientSettings = eventHubClientSettings;
_mapper = mapper;
_equipmentLocClient = new EventHubProducerClient(_eventHubClientSettings.ConnectionString, _eventHubClientSettings.EquipmentLocation);
}
}
}
Based on my speculation in comments, I'm curious if refactoring to use async/await rather than the explicit continuation in the main loop may help. Perhaps something similar to the following LinqPad snippet:
async Task Main()
{
while (true)
{
var message = await Task.Factory.StartNew(() => GetText());
var events = new[] { new EventData(Encoding.UTF8.GetBytes(message)) };
await Send(events).ConfigureAwait(false);
}
}
public EventHubProducerClient client = new EventHubProducerClient("<< CONNECTION STRING >>");
public async Task Send(EventData[] events)
{
try
{
await client.SendAsync(events).ConfigureAwait(false);
"Sent".Dump();
}
catch (Exception ex)
{
ex.Dump();
}
}
public string GetText()
{
Thread.Sleep(250);
return "Test";
}
If you're set on keeping the continuation, I wonder if a slight structural refactoring in the continuation may help, both to push up creation of the events and to honor the await statements. Perhaps something similar to the following LinqPad snippet:
async Task Main()
{
while(true)
{
var t = Task.Factory.StartNew(() => GetText());
var _ = t.ContinueWith(async q =>
{
var events = new[] { new EventData(Encoding.UTF8.GetBytes(t.Result)) };
await Send(events).ConfigureAwait(false);
});
await Task.Yield();
}
}
public EventHubProducerClient client = new EventHubProducerClient("<< CONNECTION STRING >>");
public async Task Send(EventData[] events)
{
try
{
await client.SendAsync(events).ConfigureAwait(false);
"Sent".Dump();
}
catch (Exception ex)
{
ex.Dump();
}
}
public string GetText()
{
Thread.Sleep(250);
return "Test";
}
I'm working on a little console application that selects messages from a database queue
and forwards the messages to a rest api (ASP.NET Web Api).
In general the application does the following steps:
Get the number of pending messages
Load the last pending messages
Post the message to the rest api
Remove the message from the database
To make the program more flexible and to have the ability to process every single message
in a separate database transcation the steps 2 - 3 will be executed as tasks.
This means if there are four messages in the database, we'll have four tasks that
should run nearly parallel and process the messages.
This is what the code looks like:
Database message
public class DatabaseMessage
{
public string Message { get; set; }
}
UnitOfWork (Interface IUnitOfWork)
public class UnitOfWork
{
// ... extermely simplified
public int GetNumberOfPendingMessages() { ... }
public DatabaseMessage GetNextPendingMessage() { ... }
public void DeleteMessage(DatabaseMessage message) { ... }
}
HttpService (Interface IHttpService)
public class HttpService
{
private readonly HttpClient _httpClient;
public HttpService()
{
_httpClient = new HttpClient();
/* Some initalization stuff for the HttpClient object */
}
public async Task<HttpResponse> PostMessage(DatabaseMessage message)
{
var content = /* Create content object */
return await _httpClient.PostAsync(..., content);
}
}
MessageProcessingService (Interface IMessageProcessingService)
public class MessageProcessingService
{
private readonly IHttpService _httpService;
private readonly Semaphore _databaseProcessingSemaphore;
public MessageProcessingService(IHttpService httpService)
{
_httpService = httpService;
}
public async Task ProcessDatabaseMessages()
{
var unitOfWork = new UnitOfWork();
var numberOfPendingMessages = unitOfWork.GetNumberOfPendingMessages();
var messageProcessingTasks = new List<Task>();
for(int t = 0; t < numberOfPendingMessages; t++)
{
messageProcessingTasks.Add(new Task(() => {
ProcessMessageAsTask();
}));
}
var continuationHandler = Task.WhenAll(messageProcessingTasks);
messageProcessingTasks.ForEach(e => e.Start());
await continuationHandler;
}
private void ProcessMessageAsTask()
{
// Single unit of work for each tasks
var unitOfWork = new UnitOfWork();
try{
// Starting a database transaction
unitOfWork.StartTransaction();
_databaseProcessingSemaphore.OnWait();
var message = unitOfWork.GetNextPendingMessage();
_databaseProcessingSemaphore.Release();
if(message != null)
{
var response = _httpService.PostMessage(message).Result;
if(response == HttpStatus.OK)
{
unitOfWork.DeleteMessage(message);
unitOfWork.Commit();
}
else
{
unitOfWork.Rollback();
}
}
else
{
unitOfWork.Commit();
}
}
catch(Exception ex)
{
unitOfWork.Rollback();
// Further error handling...
}
}
}
For better understanding, the HttpClient object is created and managed by Unity and is injected
into the MessageProcessingService object. The HttpClient is held as singleton in the container.
I'm facing now the problem that the call of the method _httpService.PostMessage(). For example, if there are five
messages in the message queue, the call fails five times with an exception that tells me that an task has been canceled.
My question is now what is the problem with PostAsync call of the .NET HttpClient? Is the issue caused by the .Result option or would
it be better to create a new instance of the HttpClient for each message processing task?
Or is there a general problem with the architecture with tasks and the processing of rest api calls?
Update 2018-04-04 - 08:09
I've now made the method ProcessMessageAsTask async and I'm awaiting now the call of the HttpService.
But now I don't get any exception at all. In the ressource monitor and while debugging I can see that all tasks reach the call of the HttpClient (return await _httpClient.PostAsync(..., content);)
But there is no exception nor will the messages be posted. But I don't get any exceptions. The program will be closed immediately after the calls of the HttpClient. All futher statements were not processed.
Changes:
public async Task ProcessDatabaseMessages()
{
var unitOfWork = new UnitOfWork();
var numberOfPendingMessages = unitOfWork.GetNumberOfPendingMessages();
var messageProcessingTasks = new List<Task>();
for(int t = 0; t < numberOfPendingMessages; t++)
{
messageProcessingTasks.Add(new Task(async () => {
await ProcessMessageAsTask();
}));
}
var continuationHandler = Task.WhenAll(messageProcessingTasks);
messageProcessingTasks.ForEach(e => e.Start());
await continuationHandler;
}
private async Task ProcessMessageAsTask()
{
// Single unit of work for each tasks
var unitOfWork = new UnitOfWork();
try{
// Starting a database transaction
unitOfWork.StartTransaction();
_databaseProcessingSemaphore.OnWait();
var message = unitOfWork.GetNextPendingMessage();
_databaseProcessingSemaphore.Release();
if(message != null)
{
var response = await _httpService.PostMessage(message);
if(response == HttpStatus.OK)
{
unitOfWork.DeleteMessage(message);
unitOfWork.Commit();
}
else
{
unitOfWork.Rollback();
}
}
else
{
unitOfWork.Commit();
}
}
catch(Exception ex)
{
unitOfWork.Rollback();
// Further error handling...
}
}
I have a problem, and I am not quite sure how to solve this, except for making my Akka Actor not have async methods.
Here is my Actor Code:
public class AggregatorActor : ActorBase, IWithUnboundedStash
{
public IStash Stash { get; set; }
private AggregatorTimer _aggregatorTimer;
private IActorSystemSettings _settings;
private AccountSummary _accountResponse;
private ContactDetails _contactResponse;
private AnalyticDetails _analyticsResponse;
private FinancialDetails _financialResponse;
private ActorSelection _accountActor;
private ActorSelection _contactActor;
private ActorSelection _analyticsActor;
private ActorSelection _financialActor;
public AggregatorActor(IActorSystemSettings settings) : base(settings)
{
_accountActor = Context.System.ActorSelection(ActorPaths.AccountActorPath);
_contactActor = Context.System.ActorSelection(ActorPaths.ContactActorPath);
_analyticsActor = Context.System.ActorSelection(ActorPaths.AnalyticsActorPath);
_financialActor = Context.System.ActorSelection(ActorPaths.FinancialActorPath);
_settings = settings;
}
#region Public Methods
public override void Listening()
{
ReceiveAsync<ProfilerMessages.ProfilerBase>(async x => await HandleMessageAsync(x));
}
private void Busy()
{
Receive<ProfilerMessages.ProfilerBase>(x => Stash.Stash());
}
private void Aggregate()
{
try
{
Context.Sender.Tell(AggregatedSummaryResponse.Instance(_accountResponse, _contactResponse, _analyticsResponse, _financialResponse));
}
catch (Exception ex)
{
ExceptionHandler(ex);
}
}
public override async Task HandleMessageAsync(object msg)
{
//if is summary, generate new isntance of AggregatorTimer in _eventHandlerCollection.
if (msg is ProfilerMessages.GetSummary)
{
//Become busy. Stash
Become(Busy);
//Handle different requests
var clientId = (msg as ProfilerMessages.GetSummary).ClientId;
await HandleSummaryRequest(clientId);
}
}
private async Task HandleSummaryRequest(string clientId)
{
try
{
var accountMsg = new AccountMessages.GetAggregatedData(clientId);
_accountResponse = (await _accountActor.Ask<Messages.AccountMessages.AccountResponseAll>(accountMsg, TimeSpan.FromSeconds(_settings.NumberOfSecondsToWaitForResponse))).AccountDetails;
//Need to uncomment this
var contactMsg = new ContactMessages.GetAggregatedContactDetails(clientId);
_contactResponse = (await _contactActor.Ask<Messages.ContactMessages.ContactResponse>(contactMsg, TimeSpan.FromSeconds(_settings.NumberOfSecondsToWaitForResponse))).ContactDetails;
var analyticMsg = new AnalyticsMessages.GetAggregatedAnalytics(clientId);
_analyticsResponse = (await _analyticsActor.Ask<Messages.AnalyticsMessages.AnalyticsResponse>(analyticMsg, TimeSpan.FromSeconds(_settings.NumberOfSecondsToWaitForResponse))).AnalyticDetails;
var financialMsg = new FinancialMessages.GetAggregatedFinancialDetails(clientId);
_financialResponse = (await _financialActor.Ask<Messages.FinancialMessages.FinancialResponse>(financialMsg, TimeSpan.FromSeconds(_settings.NumberOfSecondsToWaitForResponse))).FinancialDetails;
//Start new timer
_aggregatorTimer = new AggregatorTimer(_settings.NumberOfSecondsToWaitForResponse);
_aggregatorTimer.TimeElapsed += _aggregatorTimer_TimeElapsed;
}
catch (Exception ex)
{
ExceptionHandler(ex);
}
}
//Event that is raised when an external timers time elapsed.
private async void _aggregatorTimer_TimeElapsed(object sender, ElapsedTimeHandlerArg e)
{
Aggregate();
_aggregatorTimer = null;
_accountResponse = null;
_contactResponse = null;
_analyticsResponse = null;
_financialResponse = null;
//Unstash
Stash.Unstash();
//Start listening again
Become(Listening);
}
#endregion
}
Inside the _aggregatorTimer_TimeElapsed event, I call the await Aggregate function, but the following exception is thrown.
There is no active ActorContext, this is most likely due to use of async operations from within this actor
I think this is caused by the fact that the Aggregate function tries to Tell() the Sender about the responses that are aggregated, but those Tasksare not yet completed? I might be completely wrong, but I have no idea why this is thrown.
I'm not sure what do you even need an AggregatorTimer for - in akka you have a Context.System.Scheduler object, which can be used to schedule events going to happen in the future.
Another thing is that you probably shouldn't execute logic inside event handlers. If you really need them in your code, it's better to limit them only to send a message, once an event gets triggered i.e.:
Receive<TimedOut>(_ => /* handle timeout message */);
var self = Self; // bind current self to a variable, so it won't change
_aggregatorTimer.TimeElapsed += (sender, e) => self.Tell(new TimedOut(), self);
In my service I currently have a few tasks and a ReportProgress method that continually updates a List. How can I return that list to my client host application?
Service side:
public async void BeginSync(string dbId)
{
var progressIndicator = new Progress<string>(ReportSyncProgress);
var output = await BeginSyncTaskAsync(dbId, progressIndicator);
}
...within the task I have a progress Report in a loop:
while ((output = process.StandardOutput.ReadLine()) != null)
{
progress.Report(output);
}
...and here is my report method:
public void ReportSyncProgress(string value)
{
// report by appending to global list
progressOutput.Add(value);
}
progressOutput is a List and I need my client to receive that in real time as it is updated.
Thank you!
Because Rest services don't have sessions you can't make normal WCF callback method. Instead what you will need to do is pass in some kind of token and then use that token to get the progress information.
private static ConcurrentDictionary<Guid, ConcurrentQueue<string>> _progressInfo;
//You should never do "async void" WCF can handle using tasks and having Async suffixes.
//see https://blogs.msdn.microsoft.com/endpoint/2010/11/12/simplified-asynchronous-programming-model-in-wcf-with-asyncawait/
public async Task BeginSyncAsync(string dbId, Guid progressKey)
{
if (!_progressInfo.TryAdd(progressKey, new ConcurrentQueue<string>()))
{
throw new InvalidOperationException("progress key is in use");
}
var progressIndicator = new Progress<string>((value) => ReportSyncProgress(value, progressKey));
try
{
var output = await BeginSyncTaskAsync(dbId, progressIndicator);
}
finally
{
//Remove progress list
ConcurrentQueue<string> temp;
_progressInfo.TryRemove(progressKey, out temp);
}
}
public List<string> GetSyncProgress(Guid progressKey)
{
ConcurrentQueue<string> progressOutput;
if (!_progressInfo.TryGetValue(progressKey, out progressOutput))
{
//the key did not exist, retun null;
return null;
}
//transform the queue to a list and return it.
return progressOutput.ToList();
}
private void ReportSyncProgress(string value, Guid progressKey)
{
ConcurrentQueue<string> progressOutput;
if (!_progressInfo.TryGetValue(progressKey, out progressOutput))
{
//the key did not exist, progress is being reported for a completed item... odd.
return;
}
//This is the requests specific queue of output.
progressOutput.Enqueue(value);
}
I'm currently working on a a project and I have a need to queue some jobs for processing, here's the requirement:
Jobs must be processed one at a time
A queued item must be able to be waited on
So I want something akin to:
Task<result> QueueJob(params here)
{
/// Queue the job and somehow return a waitable task that will wait until the queued job has been executed and return the result.
}
I've tried having a background running task that just pulls items off a queue and processes the job, but the difficulty is getting from a background task to the method.
If need be I could go the route of just requesting a completion callback in the QueueJob method, but it'd be great if I could get a transparent Task back that allows you to wait on the job to be processed (even if there are jobs before it in the queue).
You might find TaskCompletionSource<T> useful, it can be used to create a Task that completes exactly when you want it to. If you combine it with BlockingCollection<T>, you will get your queue:
class JobProcessor<TInput, TOutput> : IDisposable
{
private readonly Func<TInput, TOutput> m_transform;
// or a custom type instead of Tuple
private readonly
BlockingCollection<Tuple<TInput, TaskCompletionSource<TOutput>>>
m_queue =
new BlockingCollection<Tuple<TInput, TaskCompletionSource<TOutput>>>();
public JobProcessor(Func<TInput, TOutput> transform)
{
m_transform = transform;
Task.Factory.StartNew(ProcessQueue, TaskCreationOptions.LongRunning);
}
private void ProcessQueue()
{
Tuple<TInput, TaskCompletionSource<TOutput>> tuple;
while (m_queue.TryTake(out tuple, Timeout.Infinite))
{
var input = tuple.Item1;
var tcs = tuple.Item2;
try
{
tcs.SetResult(m_transform(input));
}
catch (Exception ex)
{
tcs.SetException(ex);
}
}
}
public Task<TOutput> QueueJob(TInput input)
{
var tcs = new TaskCompletionSource<TOutput>();
m_queue.Add(Tuple.Create(input, tcs));
return tcs.Task;
}
public void Dispose()
{
m_queue.CompleteAdding();
}
}
I would go for something like this:
class TaskProcessor<TResult>
{
// TODO: Error handling!
readonly BlockingCollection<Task<TResult>> blockingCollection = new BlockingCollection<Task<TResult>>(new ConcurrentQueue<Task<TResult>>());
public Task<TResult> AddTask(Func<TResult> work)
{
var task = new Task<TResult>(work);
blockingCollection.Add(task);
return task; // give the task back to the caller so they can wait on it
}
public void CompleteAddingTasks()
{
blockingCollection.CompleteAdding();
}
public TaskProcessor()
{
ProcessQueue();
}
void ProcessQueue()
{
Task<TResult> task;
while (blockingCollection.TryTake(out task))
{
task.Start();
task.Wait(); // ensure this task finishes before we start a new one...
}
}
}
Depending on the type of app that is using it, you could switch out the BlockingCollection/ConcurrentQueue for something simpler (eg just a plain queue). You can also adjust the signature of the "AddTask" method depending on what sort of methods/parameters you will be queueing up...
Func<T> takes no parameters and returns a value of type T. The jobs are run one by one and you can wait on the returned task to get the result.
public class TaskQueue
{
private Queue<Task> InnerTaskQueue;
private bool IsJobRunning;
public void Start()
{
Task.Factory.StartNew(() =>
{
while (true)
{
if (InnerTaskQueue.Count > 0 && !IsJobRunning)
{
var task = InnerTaskQueue.Dequeue()
task.Start();
IsJobRunning = true;
task.ContinueWith(t => IsJobRunning = false);
}
else
{
Thread.Sleep(1000);
}
}
}
}
public Task<T> QueueJob(Func<T> job)
{
var task = new Task<T>(() => job());
InnerTaskQueue.Enqueue(task);
return task;
}
}