I'm trying to create an API that consumes various topics.
For this, I'm trying to multi-thread things, so that the whole thing can be scalable into multiple APIs, later on, but that's very besides the point.
I'm using ASP.net Core 4.0, if that's got anything to do with it. Entity Framework as well.
My problem is based on my connection to my Mosquitto server being broken without throwing an exception or anything of the like, after a minute or so. It doesn't matter how big the messages are, or how many are exchanged. I have no idea of how I can create a callback or anything of the kind to know what's going on with my connection. Can anyone help?
I'll link the code I use to establish a connection and subscribe to a connection below. Using the Subscribe method or doing it manually also changes nothing. I'm at a loss, here.
Thanks in advance!
Main.cs:
Task.Factory.StartNew(() => DataflowController.ResumeQueuesAsync());
BuildWebHost(args).Run();
DataflowController.cs:
public static Boolean Subscribe(String topic)
{
Console.WriteLine("Hello from " + topic);
MqttClient mqttClient = new MqttClient(brokerAddress);
byte code = mqttClient.Connect(Guid.NewGuid().ToString());
// Register to message received
mqttClient.MqttMsgPublishReceived += client_recievedMessageAsync;
string clientId = Guid.NewGuid().ToString();
mqttClient.Connect(clientId);
// Subscribe to topic
mqttClient.Subscribe(new String[] { topic }, new byte[] { MqttMsgBase.QOS_LEVEL_EXACTLY_ONCE });
System.Console.ReadLine();
return true;
}
public static async Task ResumeQueuesAsync()
{
var mongoClient = new MongoClient(connectionString);
var db = mongoClient.GetDatabase(databaseName);
var topics = db.GetCollection<BsonDocument>(topicCollection);
var filter = new BsonDocument();
List<BsonDocument> result = topics.Find(filter).ToList();
var resultSize = result.Count;
Task[] subscriptions = new Task[resultSize];
MqttClient mqttClient = new MqttClient(brokerAddress);
byte code = mqttClient.Connect(Guid.NewGuid().ToString());
// Register to message received
mqttClient.MqttMsgPublishReceived += client_recievedMessageAsync;
string clientId = Guid.NewGuid().ToString();
mqttClient.Connect(clientId);
int counter = 0;
foreach(var doc in result)
{
subscriptions[counter] = new Task(() =>
{
Console.WriteLine("Hello from " + doc["topic"].ToString());
// Subscribe to topic
mqttClient.Subscribe(new String[] { doc["topic"].ToString() }, new byte[] { MqttMsgBase.QOS_LEVEL_EXACTLY_ONCE });
System.Console.ReadLine();
});
counter++;
}
foreach(Task task in subscriptions)
{
task.Start();
}
}
static async void client_recievedMessageAsync(object sender, MqttMsgPublishEventArgs e)
{
// Handle message received
var message = System.Text.Encoding.Default.GetString(e.Message);
var topic = e.Topic;
var id = topic.Split("/")[2];
BsonDocument doc = new BsonDocument {
{"Plug ID", id },
{"Consumption", message }
};
await Save(doc, "smartPDM_consumption");
System.Console.WriteLine("Message received from " + topic + " : " + message);
}
This line was the issue:
byte code = mqttClient.Connect(Guid.NewGuid().ToString());
Deleted it, and it just worked.
Related
I am trying to send SMS to clients from transactional data. It works fine but sometimes the staging table gets tanked due to huge data load. I was thinking if there is any way to queue data in a First-In-First-Out (FIFO) order. I am still learning C#. Please help me to implement the following code in a smart way.
I am sharing what I did.
private static void Main(string[] args)
{
var datetime = DateTime.Now;
var hr = datetime.Hour;
var mm = datetime.Minute + 1;
var ss = double.Parse(ConfigurationManager.AppSettings["SMSTIME"]);
// For Interval in Seconds
// This Scheduler will start at current time + 1 minute and call after every 3 Seconds
// IntervalInSeconds(start_hour, start_minute, seconds)
SchedulerTasks.IntervalInSeconds(hr, mm, ss,
() =>
{
CreateNewSms.Instance.PostNewSmsText();
});
Console.ReadLine();
}
public async void PostNewSmsText()
{
var smsDt = await Task.Run(() => DataAccessLayer.Instance.InsertSmsDataToLocalDb());
if (smsDt.Rows.Count != 0)
{
var smsData = GetSmsData(smsDt);
var xmlRespDoc = new XmlDocument();
await Task.Run(() =>
{
foreach (var sms in smsData)
{
var smsText = GetSmsText(sms.Channel, sms.AccountNumber, sms.Amount, sms.DrCr, sms.CurrentBalance, sms.BranchName,
sms.CurrencyName, sms.TrnDescription, sms.UbsTrnDateTime);
var smsSubmitTime = DateTime.Now.ToString(CultureInfo.InvariantCulture);
var response = ExecuteSmsService.Instance.PostSmsText(sms.MobileNo, smsText, sms.TrnRefNo);
var smsDeliveryTime = DateTime.Now.ToString(CultureInfo.InvariantCulture);
var operatorInfo = ConfigurationManager.AppSettings["Operator"];
xmlRespDoc.LoadXml(response);
var smsCsmsId = xmlRespDoc.SelectSingleNode("//SMSINFO/CSMSID")?.InnerText;
var smsRefNo = xmlRespDoc.SelectSingleNode("//SMSINFO/REFERENCEID")?.InnerText;
var smsDeliveryStatus = xmlRespDoc.SelectSingleNode("//SMSINFO/MSISDNSTATUS")?.InnerText;
if (smsRefNo != null) //Save response data if sms delivery successful
{
DataAccessLayer.Instance.InsertSmsResponseDetails(
sms.EntrySerialNo,
smsCsmsId,
smsRefNo,
smsText,
smsSubmitTime,
smsDeliveryTime,
true,
"Success",
operatorInfo);
DataAccessLayer.Instance.UpdateSmsStatus(sms.EntrySerialNo);
DataAccessLayer.Instance.DeleteSmsData(sms.EntrySerialNo, true);
Console.WriteLine("SMS successfully sent to: " + sms.MobileNo);
}
else
{
//If any sms delivery failed here we set a counter to retry again in next date
DataAccessLayer.Instance.SetSmsCounter(sms.EntrySerialNo);
//Save response data if sms delivery unsuccessful with failed status
DataAccessLayer.Instance.InsertSmsResponseDetails(
sms.EntrySerialNo,
smsCsmsId,
null,
smsText,
smsSubmitTime,
smsDeliveryTime,
false,
smsDeliveryStatus,
operatorInfo);
DataAccessLayer.Instance.DeleteSmsData(sms.EntrySerialNo, false);
Console.WriteLine("Invalid mobile no: " + sms.MobileNo);
}
}
});
}
else
{
Console.WriteLine("Waiting for transactions.....");
}
}
I was searching on Queue Implementations and found this but confused how to use it here in my code.
public class SendSmsQueue
{
private ConcurrentQueue<object> _jobs = new ConcurrentQueue<object>();
public SendSmsQueue()
{
var thread = new Thread(new ThreadStart(OnStart));
thread.IsBackground = true;
thread.Start();
}
public void Enqueue(object job)
{
_jobs.Enqueue(job);
}
private void OnStart()
{
while (true)
{
if (_jobs.TryDequeue(out object result))
{
Console.WriteLine(result);
}
}
}
}
I am doing some performance tests on ZeroMQ in order to compare it with others like RabbitMQ and ActiveMQ.
In my broadcast tests and to avoid "The Dynamic Discovery Problem" as referred by ZeroMQ documentation I have used a proxy. In my scenario, I am using 50 concurrent publishers each one sending 500 messages with 1ms delay between sends. Each message is then read by 50 subscribers. And as I said I am losing messages, each of the subscribers should receive a total of 25000 messages and they are each receiving between 5000 and 10000 messages only.
I am using Windows and C# .Net client clrzmq4 (4.1.0.31).
I have already tried some solutions that I found on other posts:
I have set linger to TimeSpan.MaxValue
I have set ReceiveHighWatermark to 0 (as it is presented as infinite, but I have tried also Int32.MaxValue)
I have set checked for slow start receivers, I made receivers start some seconds before publishers
I had to make sure that no garbage collection is made to the socket instances (linger should do it but to make sure)
I have a similar scenario (with similar logic) using NetMQ and it works fine. The other scenario does not use security though and this one does (and that's also the reason why I use clrzmq in this one because I need client authentication with certificates that is not yet possible on NetMQ).
EDIT:
public class MCVEPublisher
{
public void publish(int numberOfMessages)
{
string topic = "TopicA";
ZContext ZContext = ZContext.Create();
ZSocket publisher = new ZSocket(ZContext, ZSocketType.PUB);
//Security
// Create or load certificates
ZCert serverCert = Main.GetOrCreateCert("publisher");
var actor = new ZActor(ZContext, ZAuth.Action, null);
actor.Start();
// send CURVE settings to ZAuth
actor.Frontend.Send(new ZFrame("VERBOSE"));
actor.Frontend.Send(new ZMessage(new List<ZFrame>()
{ new ZFrame("ALLOW"), new ZFrame("127.0.0.1") }));
actor.Frontend.Send(new ZMessage(new List<ZFrame>()
{ new ZFrame("CURVE"), new ZFrame(".curve") }));
publisher.CurvePublicKey = serverCert.PublicKey;
publisher.CurveSecretKey = serverCert.SecretKey;
publisher.CurveServer = true;
publisher.Linger = TimeSpan.MaxValue;
publisher.ReceiveHighWatermark = Int32.MaxValue;
publisher.Connect("tcp://127.0.0.1:5678");
Thread.Sleep(3500);
for (int i = 0; i < numberOfMessages; i++)
{
Thread.Sleep(1);
var update = $"{topic} {"message"}";
using (var updateFrame = new ZFrame(update))
{
publisher.Send(updateFrame);
}
}
//just to make sure it does not end instantly
Thread.Sleep(60000);
//just to make sure publisher is not garbage collected
ulong Affinity = publisher.Affinity;
}
}
public class MCVESubscriber
{
private ZSocket subscriber;
private List<string> prints = new List<string>();
public void read()
{
string topic = "TopicA";
var context = new ZContext();
subscriber = new ZSocket(context, ZSocketType.SUB);
//Security
ZCert serverCert = Main.GetOrCreateCert("xpub");
ZCert clientCert = Main.GetOrCreateCert("subscriber");
subscriber.CurvePublicKey = clientCert.PublicKey;
subscriber.CurveSecretKey = clientCert.SecretKey;
subscriber.CurveServer = true;
subscriber.CurveServerKey = serverCert.PublicKey;
subscriber.Linger = TimeSpan.MaxValue;
subscriber.ReceiveHighWatermark = Int32.MaxValue;
// Connect
subscriber.Connect("tcp://127.0.0.1:1234");
subscriber.Subscribe(topic);
while (true)
{
using (var replyFrame = subscriber.ReceiveFrame())
{
string messageReceived = replyFrame.ReadString();
messageReceived = Convert.ToString(messageReceived.Split(' ')[1]);
prints.Add(messageReceived);
}
}
}
public void PrintMessages()
{
Console.WriteLine("printing " + prints.Count);
}
}
public class Main
{
static void Main(string[] args)
{
broadcast(500, 50, 50, 30000);
}
public static void broadcast(int numberOfMessages, int numberOfPublishers, int numberOfSubscribers, int timeOfRun)
{
new Thread(() =>
{
using (var context = new ZContext())
using (var xsubSocket = new ZSocket(context, ZSocketType.XSUB))
using (var xpubSocket = new ZSocket(context, ZSocketType.XPUB))
{
//Security
ZCert serverCert = GetOrCreateCert("publisher");
ZCert clientCert = GetOrCreateCert("xsub");
xsubSocket.CurvePublicKey = clientCert.PublicKey;
xsubSocket.CurveSecretKey = clientCert.SecretKey;
xsubSocket.CurveServer = true;
xsubSocket.CurveServerKey = serverCert.PublicKey;
xsubSocket.Linger = TimeSpan.MaxValue;
xsubSocket.ReceiveHighWatermark = Int32.MaxValue;
xsubSocket.Bind("tcp://*:5678");
//Security
serverCert = GetOrCreateCert("xpub");
var actor = new ZActor(ZAuth.Action0, null);
actor.Start();
// send CURVE settings to ZAuth
actor.Frontend.Send(new ZFrame("VERBOSE"));
actor.Frontend.Send(new ZMessage(new List<ZFrame>()
{ new ZFrame("ALLOW"), new ZFrame("127.0.0.1") }));
actor.Frontend.Send(new ZMessage(new List<ZFrame>()
{ new ZFrame("CURVE"), new ZFrame(".curve") }));
xpubSocket.CurvePublicKey = serverCert.PublicKey;
xpubSocket.CurveSecretKey = serverCert.SecretKey;
xpubSocket.CurveServer = true;
xpubSocket.Linger = TimeSpan.MaxValue;
xpubSocket.ReceiveHighWatermark = Int32.MaxValue;
xpubSocket.Bind("tcp://*:1234");
using (var subscription = ZFrame.Create(1))
{
subscription.Write(new byte[] { 0x1 }, 0, 1);
xpubSocket.Send(subscription);
}
Console.WriteLine("Intermediary started, and waiting for messages");
// proxy messages between frontend / backend
ZContext.Proxy(xsubSocket, xpubSocket);
Console.WriteLine("end of proxy");
//just to make sure it does not end instantly
Thread.Sleep(60000);
//just to make sure xpubSocket and xsubSocket are not garbage collected
ulong Affinity = xpubSocket.Affinity;
int ReceiveHighWatermark = xsubSocket.ReceiveHighWatermark;
}
}).Start();
Thread.Sleep(5000); //to make sure proxy started
List<MCVESubscriber> Subscribers = new List<MCVESubscriber>();
for (int i = 0; i < numberOfSubscribers; i++)
{
MCVESubscriber ZeroMqSubscriber = new MCVESubscriber();
new Thread(() =>
{
ZeroMqSubscriber.read();
}).Start();
Subscribers.Add(ZeroMqSubscriber);
}
Thread.Sleep(10000);//to make sure all subscribers started
for (int i = 0; i < numberOfPublishers; i++)
{
MCVEPublisher ZeroMqPublisherBroadcast = new MCVEPublisher();
new Thread(() =>
{
ZeroMqPublisherBroadcast.publish(numberOfMessages);
}).Start();
}
Thread.Sleep(timeOfRun);
foreach (MCVESubscriber Subscriber in Subscribers)
{
Subscriber.PrintMessages();
}
}
public static ZCert GetOrCreateCert(string name, string curvpath = ".curve")
{
ZCert cert;
string keyfile = Path.Combine(curvpath, name + ".pub");
if (!File.Exists(keyfile))
{
cert = new ZCert();
Directory.CreateDirectory(curvpath);
cert.SetMeta("name", name);
cert.Save(keyfile);
}
else
{
cert = ZCert.Load(keyfile);
}
return cert;
}
}
This code also produces the expected number of messages when security is disabled, but when turned on it doesn't.
Does someone know another thing to check? Or has it happened to anyone before?
Thanks
I want to retrieve ticks from Poloniex in real time. They use wamp for that. I installed via nugget WampSharp and found this code :
static async void MainAsync(string[] args)
{
var channelFactory = new DefaultWampChannelFactory();
var channel = channelFactory.CreateMsgpackChannel("wss://api.poloniex.com", "realm1");
await channel.Open();
var realmProxy = channel.RealmProxy;
Console.WriteLine("Connection established");
int received = 0;
IDisposable subscription = null;
subscription =
realmProxy.Services.GetSubject("ticker")
.Subscribe(x =>
{
Console.WriteLine("Got Event: " + x);
received++;
if (received > 5)
{
Console.WriteLine("Closing ..");
subscription.Dispose();
}
});
Console.ReadLine();
}
but no matter at the await channel.open() I have the following error : HHTP 502 bad gateway
Do you have an idea where is the problem
thank you in advance
The Poloniex service seems not to be able to handle so many connections. That's why you get the HTTP 502 bad gateway error. You can try to use the reconnector mechanism in order to try connecting periodically.
static void Main(string[] args)
{
var channelFactory = new DefaultWampChannelFactory();
var channel = channelFactory.CreateJsonChannel("wss://api.poloniex.com", "realm1");
Func<Task> connect = async () =>
{
await Task.Delay(30000);
await channel.Open();
var tickerSubject = channel.RealmProxy.Services.GetSubject("ticker");
var subscription = tickerSubject.Subscribe(evt =>
{
var currencyPair = evt.Arguments[0].Deserialize<string>();
var last = evt.Arguments[1].Deserialize<decimal>();
Console.WriteLine($"Currencypair: {currencyPair}, Last: {last}");
},
ex => {
Console.WriteLine($"Oh no! {ex}");
});
};
WampChannelReconnector reconnector =
new WampChannelReconnector(channel, connect);
reconnector.Start();
Console.WriteLine("Press a key to exit");
Console.ReadKey();
}
This is based on this code sample.
Get rid of the Console.WriteLine it is interfering with your code.
I am new to WebSockets (this AM) and have set up a WCF WebSocket app that works when doing a trivial example I found online (http://www.codeproject.com/Articles/619343/Using-WebSocket-in-NET-Part).
I added Entity Framework and as soon as I add code to try to access data the process (just sending a message back and forth) no longer works.
Could there be some fundamental concept I could be missing?
Does anyone have any good ideas for troubleshooting?
namespace PBWebSocket
{
public class PBWebSocket : IBWebSocket
{
private SPEntities db = new SPEntities();
public async Task SendMessageToServer(Message msg)
{
var callback = OperationContext.Current.GetCallbackChannel<IPBCallback>();
if (msg.IsEmpty || ((IChannel)callback).State != CommunicationState.Opened)
{
return;
}
byte[] body = msg.GetBody<byte[]>();
string msgTextFromClient = Encoding.UTF8.GetString(body);
var reqId = Int32.Parse(msgTextFromClient);
// *** The below line breaks it ***
var req = db.Requests.Where(r => r.Id == 164).FirstOrDefault();
reqId = reqId + 2;
Message newMsg = ByteStreamMessage.CreateMessage(
new ArraySegment<byte>(Encoding.UTF8.GetBytes(reqId.ToString())));
newMsg.Properties["WebSocketMessageProperty"] =
new WebSocketMessageProperty
{ MessageType = WebSocketMessageType.Text };
await callback.SendMessageToClient(newMsg);
}
}
}
I am using push sharp version PushSharp 4.0.4.
I am using it in a windows application.
I have three main methods
1- BroadCastToAll
2- BrodcatsToIOS
3- BrodcatsToAndriod
I have a button calld send. On the click event of the button. I am calling the
BroadCastToAll function.
private void btnSend_Click(object sender, EventArgs e)
{
var url = "www.mohammad-jouhari.com"
var promotion = new Promotion ();
BroadCastToAll(promotion, url);
}
Here is the BrodcastToAll Function
public void BroadCastToAll(Promotion promotion, string url)
{
var deviceCatalogs = GetDeviceCatalog();
BroadCastToIOS(promotion, url, deviceCatalogs.Where(d => d.OS == "IOS").ToList());
BroadCastToAndriod(promotion, url, deviceCatalogs.Where(d => d.OS == "Android").ToList());
}
Here is the BrodcastToIOS Function
public void BroadCastToIOS(Promotion promotion, string url, List<DeviceCatalog> deviceCatalogs)
{
if (deviceCatalogs.Count == 0)
return;
lock (_lock)// Added this lock because there is a potential chance that PushSharp callback execute during registering devices
{
QueueAllAppleDevicesForNotification(promotion, url, deviceCatalogs, logsMessage);
}
}
Here is the BrodcastToAndriod Function
public void BroadCastToAndriod(Promotion promotion, string url, List<DeviceCatalog> deviceCatalogs)
{
if (deviceCatalogs.Count == 0)
return;
lock (_lock)// Added this lock because there is a potential chance that PushSharp callback execute during registering devices
{
QueueAllGcmDevicesForNotification(promotion, url, deviceCatalogs, logsMessage);
}
}
Here is the QueueAllAppleDevicesForNotification function
private void QueueAllAppleDevicesForNotification(Promotion promotion, string url, List<DeviceCatalog> deviceCatalogs)
{
var apnsServerEnviroment = UseProductionCertificate ? ApnsConfiguration.ApnsServerEnvironment.Production : ApnsConfiguration.ApnsServerEnvironment.Sandbox;
var fileService = new FileService();
var filePath = Application.StartupPath+ "/Certifcates/" + (UseProductionCertificate ? "prod.p12" : "dev.p12");
var buffer = fileService.GetFileBytes(filePath);
var config = new ApnsConfiguration(apnsServerEnviroment, buffer, APPLE_CERTIFICATE_PWD);
apnsServiceBroker = new ApnsServiceBroker(config);
apnsServiceBroker.OnNotificationFailed += (notification, aggregateEx) => {
aggregateEx.Handle (ex => {
// Log the Resposne
});
};
apnsServiceBroker.OnNotificationSucceeded += (notification) => {
// Log The Response
};
apnsServiceBroker.Start();
foreach (var deviceToken in deviceCatalogs) {
var title = GetTitle(promotion, deviceToken);
//title += DateTime.UtcNow.TimeOfDay.ToString();
var NotificationPayLoadObject = new NotificationPayLoadObjectApple();
NotificationPayLoadObject.aps.alert = title;
NotificationPayLoadObject.aps.badge = 0;
NotificationPayLoadObject.aps.sound = "default";
NotificationPayLoadObject.url = url;
var payLoad = JObject.Parse(JsonConvert.SerializeObject(NotificationPayLoadObject));
apnsServiceBroker.QueueNotification(new ApnsNotification
{
Tag = this,
DeviceToken = deviceToken.UniqueID,
Payload = payLoad
});
}
var fbs = new FeedbackService(config);
fbs.FeedbackReceived += (string deviceToken, DateTime timestamp) =>
{
// This Token is no longer avaialble in APNS
new DeviceCatalogService().DeleteExpiredIosDevice(deviceToken);
};
fbs.Check();
apnsServiceBroker.Stop();
}
And here is the QueueAllGcmDevicesForNotification
private void QueueAllGcmDevicesForNotification(Promotion promotion, string url, List<DeviceCatalog> deviceCatalogs, )
{
var config = new GcmConfiguration(ANDROID_SENDER_ID, ANDROID_SENDER_AUTH_TOKEN, ANDROID_APPLICATION_ID_PACKAGE_NAME);
gcmServiceBroker = new GcmServiceBroker(config);
gcmServiceBroker.OnNotificationFailed += (notification, aggregateEx) => {
aggregateEx.Handle (ex => {
// Log Response
return true;
});
};
gcmServiceBroker.OnNotificationSucceeded += (notification) => {
// Log Response
};
var title = GetTitle(shopexPromotion);
gcmServiceBroker.Start ();
foreach (var regId in deviceCatalogs) {
var NotificationPayLoadObject = new NotificationPayLoadObjectAndriod(url, title, "7", promotion.ImageUrl);
var payLoad = JObject.Parse(JsonConvert.SerializeObject(NotificationPayLoadObject));
gcmServiceBroker.QueueNotification(new GcmNotification
{
RegistrationIds = new List<string> {
regId.UniqueID
},
Data = payLoad
});
}
gcmServiceBroker.Stop();
}
Now When I click the send button. The event will start executing.
The BrodcastToAll function will be called. I am calling BrodcastToIOS devices first and then BrodcatsToAndriod.
Is there any way in which I can call BrodcastToIOS and wait until all the devices have been Queued and notification has been pushed by the library and the call back events fired fully then start executing the BrodcastToAndriod Fucntion ?
What lines of code I need to add ?
also Is there any way to batch the number of devices to be Queued ?
For example.
Let us say I have 1000 Devices
500 IOS
500 Andriod
Can I queue 100, 100,100,100,100 for IOS and when it's done
I queue 100,100,100,100,100 for Andriod.
Any Help is appreciated.
Thanks.
The call to broker.Stop () by default will block until all the notifications from the queue have been processed.