I have implemented a rabbitmq messaging in my application. A very weird behaviour then.
My publisher is a webservice, while my consumer is a console application. After recieving the message I ack it immediately and span a new thread to process which take like 2 seconds to finish.
But the same message is sent with the previous delivery tag incremented by one. I am using topic based routing.
What would I be doing wrong?
Subscriber:
//Create the connection factory
var connectionFactory = new ConnectionFactory()
{
HostName = host,
UserName = userName,
Password = password
};
connectionFactory.RequestedHeartbeat = 10;
connectionFactory.RequestedConnectionTimeout = 30000;
connectionFactory.AutomaticRecoveryEnabled = true;
connectionFactory.NetworkRecoveryInterval = TimeSpan.FromSeconds(10);
//connection
var connection = connectionFactory.CreateConnection();
logger.Info($"Connected to RabbitMQ {host}");
connection.ConnectionShutdown += Connection_ConnectionShutdown;
var model = connection.CreateModel();
model.BasicQos(0, 1, false);
var consumer = new QueueingBasicConsumer(model);
model.BasicConsume(queueName, false, consumer);
while (true)
{
try
{
var deliveryArgs = consumer.Queue.Dequeue();
model.BasicAck(deliveryArgs.DeliveryTag, false);
var jsonString = Encoding.Default.GetString(deliveryArgs.Body);
var itemtoprocess = jsonString.FromJson < recieved message > ();
if (deliveryArgs.Redelivered)
{
model.BasicReject(deliveryArgs.DeliveryTag, false);
}
else
{
var task = Task.Factory.StartNew(() => {
//Do work here on different thread then this one
//Call the churner to process the message
//Some long running method here to process item recieve
});
Task.WaitAll(task);
}
}
catch (EndOfStreamException ex)
{
//log
}
}
Setup:
public void Init(string exchangeName, string queueName, string routingKey = "")
{
using (IConnection connection = connectionFactory.CreateConnection())
{
using (IModel channel = connection.CreateModel())
{
channel.ExchangeDeclare(exchangeName, ExchangeType.Topic, true, false, null);
//Queue
var queue = channel.QueueDeclare(queueName, true, false, false, null);
channel.QueueBind(queue.QueueName, exchangeName, routingKey);
}
}
}
Related
Whenever rabbitMq Receives any messages it should trigger some functionality.
How can i achieve this ?
We can use receive event to do the functionality.
Below code sample shows how to do the same.
ConnectionFactory oFactory = new ConnectionFactory();
oFactory.UserName = oRabbitMQData.userName;
oFactory.Password = oRabbitMQData.password;
oFactory.VirtualHost = oRabbitMQData.virtualHost;
oFactory.HostName = oRabbitMQData.hostName;
oFactory.Port = oRabbitMQData.port;
var connection = oFactory.CreateConnection();
var channel = connection.CreateModel();
// setup signal
var signal = new ManualResetEvent(true);
oRabbitMQConnections.Add(connection);
oRabbitMQChannels.Add(channel);
var consumer = new EventingBasicConsumer(channel);
byte[] messageBody = null;
consumer.Received += (sender, args) =>
{
messageBody = args.Body;
string sMsg = UTF8Encoding.UTF8.GetString(messageBody);
// process your message or store for later
// set signal
channel.BasicAck(args.DeliveryTag, false);
// Do something
signal.Set();
};
// start consuming
channel.BasicConsume(oRabbitMQData.queueName, false, consumer);
I have a piece of code that is using rabbitMQ to manage lists of jobs over time.
As such I have a connection and a channel open to the RabbitMQ Server to do operations with these jobs. I queue jobs with the following:
public override void QueueJob(string qid, string jobId) {
this.VerifyReadyToGo();
this.CreateQueue(qid);
byte[] messageBody = Encoding.UTF8.GetBytes(jobId);
this.channel.BasicPublish(
exchange: Exchange,
routingKey: qid,
body: messageBody,
basicProperties: null
);
OLog.Debug($"Queued job {jobId} on {qid}");
}
public override string RetrieveJobID(string qid) {
this.VerifyReadyToGo();
this.CreateQueue(qid);
BasicGetResult data = this.channel.BasicGet(qid, false);
string jobData = Encoding.UTF8.GetString(data.Body);
int addCount = 0;
while (!this.jobWaitingAck.TryAdd(jobData, data.DeliveryTag)) {
// try again.
Thread.Sleep(10);
if (addCount++ > 2) {
throw new JobReceptionException("Failed to add job to waiting ack list.");
}
}
OLog.Debug($"Found job {jobData} on queue {qid} with ackId {data.DeliveryTag}");
return jobData;
}
The issue is that after any method call like this (Publish, Get, or Acknowledge) creates some sort of background thread that does not close when the channel and connection are closed.
This means that tests pass and the operations complete successfully but when the application tries to close it hangs and does not ever finish.
Here is the connect method for reference
public override void Connect() {
if (this.Connected) {
return;
}
this.factory = new ConnectionFactory {
HostName = this.config.Hostname,
Password = this.config.Password,
UserName = this.config.Username,
Port = this.config.Port,
VirtualHost = VirtualHost
};
this.connection = this.factory.CreateConnection();
this.channel = this.connection.CreateModel();
this.channel.ExchangeDeclare(
exchange: Exchange,
type: "direct",
durable: true
);
}
What can I do to correct this issue (rabbitmq client preventing application from exiting)?
I have no idea why but this change to the Connect method makes the difference:
public override void Connect() {
if (this.Connected) {
return;
}
this.factory = new ConnectionFactory {
HostName = this.config.Hostname,
Password = this.config.Password,
UserName = this.config.Username,
Port = this.config.Port,
UseBackgroundThreadsForIO = true
};
this.connection = this.factory.CreateConnection();
this.channel = this.connection.CreateModel();
this.channel.ExchangeDeclare(
exchange: Exchange,
type: "direct",
durable: true
);
}
I have used RabbitMQ for storing messages. I noticed that messages are deleted when application restart.
I have producer and consumer in same application.
Please find producer and consumer as below. I have used durable queue as well as durable message.
So if there is only one consumer of queue and it's not consume currently then queue messages are deleted. Is it so ?
Producer:
public static void PublishMessage(RequestDto message, string queueName)
{
var factory = new ConnectionFactory() { HostName = Config.RabbitMqHostName, Port = Config.RabbitMqPortNumber };
using (var connection = factory.CreateConnection())
{
using (var channel = connection.CreateModel())
{
channel.QueueDeclare(queueName, true, false, false, null);
var properties = channel.CreateBasicProperties();
properties.SetPersistent(true);
// properties.DeliveryMode = 2; I have used this too.
string serializesMessage = Utility.SerializeSoapObject(message);
var messageBytes = Encoding.UTF8.GetBytes(serializesMessage);
channel.BasicPublish("", queueName, properties , messageBytes);
Log.Info("Record added into queue : \nMessage: " + serializesMessage);
}
}
}
Consumer:
var factory = new ConnectionFactory() { HostName = Config.RabbitMqHostName, Port = Config.RabbitMqPortNumber };
using (var connection = factory.CreateConnection())
{
using (var channel = connection.CreateModel())
{
channel.QueueDeclare(Config.RabbitMqQueueName, true, false, false, null);
var consumer = new QueueingBasicConsumer(channel);
channel.BasicConsume(Config.RabbitMqQueueName, true, consumer);
while (DoProcessMessage())
{
try
{
List<RequestDto> messages = GetMessagesInBatch(consumer);
if (messages.Count > 0)
{
ProcessMessageInParallel(messages);
}
else
{
Producer.FillRequestMessages();
}
}
catch (Exception exception)
{
Log.Error("StartConsumer - Failed to process message from RabbitMq Error: " + exception.Message, exception);
}
}
}
}
}
catch (Exception exception)
{
Log.Error(exception.Message, exception);
}
private bool DoProcessMessage()
{
return Config.MaxRequestPerDayCount > 1000;
}
If anyone can help.
You seem to be passing noAck = true to the basicConsume function:
https://www.rabbitmq.com/releases/rabbitmq-java-client/v1.7.0/rabbitmq-java-client-javadoc-1.7.0/com/rabbitmq/client/Channel.html#basicConsume(java.lang.String, boolean, com.rabbitmq.client.Consumer)
In no ack mode, RabbitMQ will send the messages to the consumer and immediately delete it from the queue.
I wondering why my RabbitMQ RPC-Client always processed the dead messages after restart. _channel.QueueDeclare(queue, false, false, false, null); should disable buffers. If I overload the QueueDeclare inside the RPC-Client I can't connect to the server. Is something wrong here? Any idea how to fix this problem?
RPC-Server
new Thread(() =>
{
var factory = new ConnectionFactory { HostName = _hostname };
if (_port > 0)
factory.Port = _port;
_connection = factory.CreateConnection();
_channel = _connection.CreateModel();
_channel.QueueDeclare(queue, false, false, false, null);
_channel.BasicQos(0, 1, false);
var consumer = new QueueingBasicConsumer(_channel);
_channel.BasicConsume(queue, false, consumer);
IsRunning = true;
while (IsRunning)
{
BasicDeliverEventArgs ea;
try {
ea = consumer.Queue.Dequeue();
}
catch (Exception ex) {
IsRunning = false;
}
var body = ea.Body;
var props = ea.BasicProperties;
var replyProps = _channel.CreateBasicProperties();
replyProps.CorrelationId = props.CorrelationId;
var xmlRequest = Encoding.UTF8.GetString(body);
var messageRequest = XmlSerializer.DeserializeObject(xmlRequest, typeof(Message)) as Message;
var messageResponse = handler(messageRequest);
_channel.BasicPublish("", props.ReplyTo, replyProps,
messageResponse);
_channel.BasicAck(ea.DeliveryTag, false);
}
}).Start();
RPC-Client
public void Start()
{
if (IsRunning)
return;
var factory = new ConnectionFactory {
HostName = _hostname,
Endpoint = _port <= 0 ? new AmqpTcpEndpoint(_endpoint)
: new AmqpTcpEndpoint(_endpoint, _port)
};
_connection = factory.CreateConnection();
_channel = _connection.CreateModel();
_replyQueueName = _channel.QueueDeclare(); // Do not connect any more
_consumer = new QueueingBasicConsumer(_channel);
_channel.BasicConsume(_replyQueueName, true, _consumer);
IsRunning = true;
}
public Message Call(Message message)
{
if (!IsRunning)
throw new Exception("Connection is not open.");
var corrId = Guid.NewGuid().ToString().Replace("-", "");
var props = _channel.CreateBasicProperties();
props.ReplyTo = _replyQueueName;
props.CorrelationId = corrId;
if (!String.IsNullOrEmpty(_application))
props.AppId = _application;
message.InitializeProperties(_hostname, _nodeId, _uniqueId, props);
var messageBytes = Encoding.UTF8.GetBytes(XmlSerializer.ConvertToString(message));
_channel.BasicPublish("", _queue, props, messageBytes);
try
{
while (IsRunning)
{
var ea = _consumer.Queue.Dequeue();
if (ea.BasicProperties.CorrelationId == corrId)
{
var xmlResponse = Encoding.UTF8.GetString(ea.Body);
try
{
return XmlSerializer.DeserializeObject(xmlResponse, typeof(Message)) as Message;
}
catch(Exception ex)
{
IsRunning = false;
return null;
}
}
}
}
catch (EndOfStreamException ex)
{
IsRunning = false;
return null;
}
return null;
}
Try setting the DeliveryMode property to non-persistent (1) in your RPC-Client code like this:
public Message Call(Message message)
{
...
var props = _channel.CreateBasicProperties();
props.DeliveryMode = 1; //you might want to do this in your RPC-Server as well
...
}
AMQP Model Explained contains very useful resources, like explaining how to handle messages that end up in the dead letter queue.
Another useful note from the documentation with regards to queue durability:
Durable queues are persisted to disk and thus survive broker restarts.
Queues that are not durable are called transient. Not all scenarios
and use cases mandate queues to be durable.
Durability of a queue does not make messages that are routed to that
queue durable. If broker is taken down and then brought back up,
durable queue will be re-declared during broker startup, however, only
persistent messages will be recovered.
Note that it talks about broker restart not publisher or consumer restart.
I am working on rabbitmq and I am confused with some points Like.
I have just implemented a sample from internet which creates a queue and then it fetch
the message from that queue thereby showing it on the webpage.
Now my problem is::
Suppose My server has RabbitmQ installed and multiple users are accessing this website where I
have implemented the rabbitmq.Now, first user sends a message but to whome it will send this message?
To all the users who will open the page because the code will be common for sent message and the name
of the queue will also be same.
Suppose, first user send one message="Hello" on the queue "Queue1"
now, one other user sends another message="Hello World" on the same queue
and one more user sends a message="Hello Worl World" on the same queue.
Now nth user clicks on receive message then which message will be shown to him?
first ,second or third one?
It means we will always have a single queue for my application?
Can somebody please guide me. I am pretty much confused...
Below I am pasting the code sample I will be using for my website
//For sending the messge
protected void btnSendMail_Click(object sender, EventArgs e)
{
try
{
var factory = new ConnectionFactory();
using (var connection = factory.CreateConnection())
{
using (var channel = connection.CreateModel())
{
// ConnectionFactory factory = new ConnectionFactory() { HostName = "localhost" };
// // factory.UserName = txtUserEmail.Text.ToString();
//// factory.Password = "password";
// factory.VirtualHost = "/";
// factory.Protocol = Protocols.FromEnvironment();
// factory.HostName = "localhost";
// factory.Port = AmqpTcpEndpoint.UseDefaultPort;
// IConnection conn = factory.CreateConnection();
// using (var channel = conn.CreateModel())
// {
// channel.QueueDeclare("hello", false, false, false, null);
// string message = "Hello World!";
// var body = Encoding.UTF8.GetBytes(txtUserEmail.Text.ToString());
// channel.BasicPublish("", "hello", null, body);
// conn.Close();
// }
//Sending Message
channel.QueueDeclare("hello1", false, false, false, null);
string message = txtUserEmail.Text.ToString();
var body = Encoding.UTF8.GetBytes(message);
channel.BasicPublish("", "hello1", null, body);
//Console.WriteLine(" [x] Sent {0}", message);
//Console.ReadLine();
Label1.Text = Encoding.Default.GetString(body);
}
}
}
catch
{
}
}
//For receiving the message.
protected void btnReceive_Click(object sender, EventArgs e)
{
try
{
//var factory = new ConnectionFactory() { HostName = "localhost" };
//using (var connection = factory.CreateConnection())
//{
// using (var channel = connection.CreateModel())
// {
// channel.QueueDeclare("hello", false, false, false, null);
// BasicGetResult result = channel.BasicGet("hello", false);
// var consumer = new QueueingBasicConsumer(channel);
// channel.BasicConsume("hello", true, consumer);
// Console.WriteLine(" [*] Waiting for messages." +
// "To exit press CTRL+C");
// while (true)
// {
// var ea = (BasicDeliverEventArgs)consumer.Queue.Dequeue();
// var body = ea.Body;
// var message = Encoding.UTF8.GetString(body);
// Console.WriteLine(" [x] Received {0}", message);
// Label1.Text = message.ToString();
// }
// }
//}
var factory = new ConnectionFactory();
using (var connection = factory.CreateConnection())
{
using (var channel = connection.CreateModel())
{
bool noAck = false;
BasicGetResult result = channel.BasicGet("hello1", noAck);
if (result == null)
{
}
else
{
IBasicProperties props = result.BasicProperties;
byte[] Body = result.Body;
Label1.Text = Encoding.Default.GetString(Body);
}
}
}
}
catch
{
}
}
If you are creating a messaging system using RabbitMQ you should probably publish your messages to an exchange and then attach a queue to the exchange for each user of the site. Then have the exchange route the messages to the right user/users queue.
You need a better understanding of messaging patterns associated with the use of RabbitMQ
These tutorials would be most relevant
Publish/Subscribe
http://www.rabbitmq.com/tutorials/tutorial-three-python.html
Routing
http://www.rabbitmq.com/tutorials/tutorial-four-python.html
The tutorials are also available in c# if you need it.