Question

我一直在寻找使用Redis Pub / Sub作为RabbitMQ的替代品。

根据我的理解，Redis的pub / sub与每个订阅者保持持久连接，如果连接终止，所有未来的消息都将丢失并丢弃在地板上。

一种可能的解决方案是使用列表（和阻塞等待）将所有消息和pub / sub存储为通知机制。我认为这让我大部分都在那里，但我仍然对失败案例有一些担忧。

当订阅者死亡并重新上线时会发生什么？它应如何处理所有待处理的消息？
当系统出现格式错误的消息时，您如何处理这些异常？ DeadLetter队列？
是否有实施重试政策的标准做法？

Answer 1

当订阅者（消费者）死亡时，您的列表将继续增长，直到客户端返回。一旦达到特定限制，您的生产者可以修剪列表（从任何一方），但这是您需要在应用程序级别处理的内容。如果在每条消息中包含时间戳，则您的消费者可以根据消息的年龄采取行动，假设您有应用程序逻辑要在消息时间上强制执行。

我不确定格式错误的消息将如何进入系统，因为与Redis的连接通常是TCP及其完整性保证。但是，如果发生这种情况，可能是由于生产者层的消息编码中的错误，您可以通过保持每个生产者队列接收消费者的异常消息来提供处理错误的一般机制。

重试政策将在很大程度上取决于您的应用需求。如果您需要100％保证已收到并处理了邮件，那么您应该考虑使用Redis事务（MULTI / EXEC）来包装使用者完成的工作，这样您就可以确保客户端不会删除邮件，除非它已经完成了它的工作。如果您需要明确的确认，那么您可以在专用于生产者进程的队列上使用显式ACK消息。

如果不了解您的应用需求，很难知道如何明智地选择。通常，如果您的消息需要完整的ACID保护，那么您可能还需要使用redis事务。如果您的消息仅在及时发布时才有意义，则可能不需要进行交易。听起来好像你不能容忍丢弃的消息，所以你使用列表的方法是好的。如果您需要为邮件实现优先级队列，则可以使用排序集（Z命令）来存储邮件，使用其优先级作为分数值，以及轮询使用者。

Answer 2

我所做的是使用排序集，使用时间戳作为分数，使用数据键作为成员值。我使用最后一项的分数来检索接下来的几项，然后获得密钥。完成工作后，我将zrem和del包装在MULTI / EXEC事务中。

基本上是爱德华所说的，但是将密钥存储在有序集中的转折，因为我的消息可能非常大。

希望这有帮助！

Answer 3

如果您想要一个发布/订阅系统，订阅者在消息丢失时不会丢失消息，请考虑使用Redis Streams而不是Redis Pub / sub。

Redis Streams拥有自己的架构和Redis Pub / sub的优缺点。使用Redis Streams，订户可以发出命令：

我收到的最后一条消息是X，现在给我下一条消息; 如果没有新消息，则等待一个消息到达。

上面链接的Antirez文章是Redis流的一个很好的介绍，有更多的信息。

警告：此功能仅存在于Redis的不稳定版（beta）版本中。

Answer 4

这是我专门为此编写的一个类：

#include <iostream>
#include<string>
#include<iomanip>
using namespace std;
float calculateCharges(float hours, float &total);

int main()

{
    int MAX=3;
    float charge[MAX];
    float hours[MAX];
    float total=0;
    int i;
    int columns[]={10,9,10};

    for(i=0; i<MAX; i++)
    {
        cout << "Enter the amount of hours for customer " <<i+1<<": ";
        cin >> hours[i];
        charge[i] = calculateCharges(hours[i], total);
    }


    string border;
    border.assign(columns[0]+columns[1]+columns[2], '-');
    cout<<border<<endl
        <<left<<setw(columns[0])<<"customer"
        <<setw(columns[1])<<"hours"
        <<setw(columns[2])<<"charge"
        <<endl;

    for(i=0; i<MAX; i++)
        cout<<left<<setw(columns[0])<<i+1
            <<right<<setw(2)<<fixed<<setprecision(2)<<hours[i]
            <<setw(columns[2])<<charge[i]<<endl; 
    cout<<border<<endl;
    cout<<right<<setw(columns[0]+columns[1])<<"total: "<<left<<" $"<<total<<endl;
    return 0;

}

float calculateCharges(float hours, float &total)
{
    float amount = 20 + (5*((int)hours-3));
    if((int)hours<=3)
        return 20;
    else return amount;
}

只需初始化 import logging from redis import StrictRedis # Defaults CONNECT_TIMEOUT_SECS = 5.0 # Control how long to wait while establishing a connection REQUEST_TIMEOUT_SECS = 120.0 # Request socket timeout class RedisBaseClient(object): def __init__(self, config=None, connect_timeout_secs=CONNECT_TIMEOUT_SECS, request_timeout_secs=REQUEST_TIMEOUT_SECS): """ Load config :param config: dict, config :param connect_timeout_secs: float, re-connect timeout seconds :param request_timeout_secs: float, timeout seconds """ self.read_conn = None self.write_conn = None self.config = config or {} self.CONNECT_TIMEOUT_SECS = connect_timeout_secs self.REQUEST_TIMEOUT_SECS = request_timeout_secs self.read_connection() def _connect(self, host, port): return StrictRedis(host=host, port=port, socket_keepalive=False, retry_on_timeout=True, socket_timeout=self.REQUEST_TIMEOUT_SECS, socket_connect_timeout=self.CONNECT_TIMEOUT_SECS) def read_connection(self): """ Returns a read connection to redis cache """ if not self.read_conn: try: self.read_conn = self._connect(self.config['read_host'], self.config['read_port']) except KeyError: logging.error("RedisCache.read_connection invalid configuration") raise except Exception as e: logging.exception("RedisCache.read_connection unhandled exception {}".format(e)) raise return self.read_conn def write_connection(self): """ Returns a write connection to redis cache """ if not self.write_conn: try: self.write_conn = self._connect(self.config['write_host'], self.config['write_port']) except KeyError: logging.error("RedisCache.write_connection invalid configuration") raise except Exception as e: logging.exception("RedisCache.write_connection unhandled exception {}".format(e)) raise return self.write_conn class RedisQueue(RedisBaseClient): def get_queue_msg_count(self, q_name): """ Get queue message count Return the size of the queue (list). :param q_name: str, redis key (queue name) :return: """ try: msg_count = self.read_connection().llen(q_name) except Exception as e: # pragma: no cover msg_count = 0 logging.warning("RedisQueue.get_queue_msg_count no data for queue {}. {}".format(q_name, e)) return msg_count def is_empty(self, q_name): """ Return True if the queue is empty, False otherwise. :param q_name: str, queue name :return: bool, is empty """ return self.get_queue_msg_count(q_name) == 0 def publish(self, q_name, data): """ Publish msg/item to queue. :param q_name: str, queue name :param data: str, data (message) :return: bool, success """ try: self.write_connection().rpush(q_name, data) except Exception as e: # pragma: no cover logging.warning("RedisQueue.publish for queue {}, msg {}. {}".format(q_name, data, e)) return False return True def publish_multiple(self, q_name, data_list): """ Publish multiple msg/items to queue. :param q_name: str, queue name :param data_list: list of str, data (message) :return: bool, success """ try: self.write_connection().rpush(q_name, *data_list) except Exception as e: # pragma: no cover logging.warning("RedisQueue.publish_multiple for queue {}. {}".format(q_name, e)) return False return True def flush_queue(self, q_name): """ Flush a queue to clear work for consumer :param q_name: :return: """ try: self.write_connection().delete(q_name) except Exception as e: # pragma: no cover logging.exception("RedisQueue.flush_queue {} error {}".format(q_name, e)) return False return True def flush_queues(self, q_names): """ Flush all queues :return: bool, success """ try: self.write_connection().delete(*q_names) except Exception as e: # pragma: no cover logging.exception("RedisQueue.flush_queues {} error {}".format(q_names, e)) return False return True def get_messages(self, q_name, prefetch_count=100): """ Get messages from queue :param q_name: str, queue name :param prefetch_count: int, number of msgs to prefetch for consumer (default 1000) """ pipe = self.write_connection().pipeline() pipe.lrange(q_name, 0, prefetch_count - 1) # Get msgs (w/o pop) pipe.ltrim(q_name, prefetch_count, -1) # Trim (pop) list to new value messages, trim_success = pipe.execute() return messages def get_message(self, q_name, timeout=None): """ Pop and return an msg/item from the queue. If optional args timeout is not None (the default), block if necessary until an item is available. Allows for blocking via timeout if queue does not exist. :param q_name: str, queue name :param timeout: int, timeout wait seconds (blocking get) :return: str, message """ if timeout is not None: msg = self.read_connection().blpop(q_name, timeout=timeout) if msg: msg = msg[1] else: msg = self.read_connection().lpop(q_name) return msg def get_message_safe(self, q_name, timeout=0, processing_prefix='processing'): """ Retrieve a message but also send it to a processing queue for later acking :param q_name: str, queue name :param timeout: :param processing_prefix: :return: """ # Too bad blpoplpush does not exist # item = self.read_connection().brpoplpush(q_name, "{}:{}".format(q_name, processing_prefix), timeout=timeout) msg = self.get_message(q_name=q_name, timeout=timeout) if msg: self.write_connection().lpush("{}:{}".format(q_name, processing_prefix), msg) return msg def ack_message_safe(self, q_name, message, processing_prefix='processing'): """ Acknowledge a message has been processed :param q_name: str, queue name :param message: str, message value :param processing_prefix: str, prefix of processing queue name :return: bool, success """ self.read_connection().lrem("{}:{}".format(q_name, processing_prefix), -1, message) return True def requeue_message_safe(self, q_name, processing_prefix='processing'): """ Move unprocessed messages from processing queue to original queue for re-processing :param q_name: :param processing_prefix: :return: bool, success """ msgs = self.write_connection().lrange("{}:{}".format(q_name, processing_prefix), 0, -1) # Get all msgs if msgs: msgs = msgs[::-1] # Reverse order pipe = self.write_connection().pipeline() pipe.rpush(q_name, *msgs) pipe.ltrim("{}:{}".format(q_name, processing_prefix), 0, -1) # Cleanup pipe.execute() return True 并使用函数。我想这就是你所追求的。

Redis Pub / Sub具有可靠性

4 个答案: