RabbitMq 一对多 多实例处理 解决拥堵
开发者难免会遇到坑爹的需求,在队列 同一组数据只能串行的情况下,适用下面操作。
探究 RabbitMq work模式 与Topic 模式
直接上代码:
package com.aspire.eab.common.configuration.rabbitmq.topic;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.amqp.core.*;
import org.springframework.amqp.core.Queue;
import org.springframework.beans.factory.annotation.Qualifier;
import java.util.HashMap;
import java.util.Map;
@Configuration
public class AsynTaskLogTopicConfig {
// 异步交换机
public static final String EAB_ASYN__TOPIC = "topicExchange.asynTaskLog";
// 异步路由 多路由 匹配
public static final String EAB_ASYN__ROUTING_FIRST = "topic.asyn.routing.FIRST.";
public static final String EAB_ASYN_ROUTING_SECOND = "topic.asyn.routing.SECOND.";
public static final String EAB_ASYN_ROUTING_THIRD = "topic.asyn.routing.THIRD.";
public static final String EAB_ASYN_ROUTING_FOURTH = "topic.asyn.routing.FOURTH.";
//总路由
public static final String EAB_ASYN_ROUTING_ZERO = "topic.asyn.routing.ZERO.";
// 异步队列
public static final String EAB_ASYN_TASK_QUEUE_FIRST = "EAB_ASYN_TASK_QUEUE_FIRST";
public static final String EAB_ASYN_TASK_QUEUE_SECOND = "EAB_ASYN_TASK_QUEUE_SECOND";
public static final String EAB_ASYN_TASK_QUEUE_THIRD = "EAB_ASYN_TASK_QUEUE_THIRD";
public static final String EAB_ASYN_TASK_QUEUE_FOURTH = "EAB_ASYN_TASK_QUEUE_FOURTH";
public static final String EAB_ASYN_TASK_QUEUE_ZERO = "EAB_ASYN_TASK_QUEUE_ZERO";
// Priority
public static final String EAB_ASYN_TASK_QUEUE_PRIORITY = "EAB_ASYN_TASK_QUEUE_PRIORITY";
@Bean(EAB_ASYN_TASK_QUEUE_FIRST)
public Queue EAB_ASYN_TASK_QUEUE_FIRST(){
return new Queue(EAB_ASYN_TASK_QUEUE_FIRST);
}
@Bean(EAB_ASYN_TASK_QUEUE_SECOND)
public Queue EAB_ASYN_TASK_QUEUE_SECOND(){
return new Queue(EAB_ASYN_TASK_QUEUE_SECOND);
}
@Bean(EAB_ASYN_TASK_QUEUE_THIRD)
public Queue EAB_ASYN_TASK_QUEUE_THIRD(){
return new Queue(EAB_ASYN_TASK_QUEUE_THIRD);
}
@Bean(EAB_ASYN_TASK_QUEUE_FOURTH)
public Queue EAB_ASYN_TASK_QUEUE_FOURTH(){
return new Queue(EAB_ASYN_TASK_QUEUE_FOURTH);
}
@Bean(EAB_ASYN_TASK_QUEUE_ZERO)
public Queue EAB_ASYN_TASK_QUEUE_ZERO(){
return new Queue(EAB_ASYN_TASK_QUEUE_ZERO);
}
@Bean(EAB_ASYN_TASK_QUEUE_PRIORITY)
public Queue EAB_ASYN_TASK_QUEUE_PRIORITY() {
Map<String, Object> args= new HashMap<>();
args.put("x-max-priority", 100); //队列的最大属性参数 请参考源码
return new Queue(AsynTaskLogTopicConfig.EAB_ASYN_TASK_QUEUE_PRIORITY, true, false, false, args);
}
@Bean(EAB_ASYN__TOPIC)
public Exchange EAB_ASYN__TOPIC(){
//Topic类型的交换机,durable是持久化
return ExchangeBuilder.topicExchange(EAB_ASYN__TOPIC).durable(true).build();
}
@Bean
public Binding BINDING_EAB_ASYN_TASK_QUEUE_FIRST(@Qualifier(EAB_ASYN_TASK_QUEUE_FIRST) Queue queue,
Exchange exchange){
return BindingBuilder.bind(queue).to(exchange).with(EAB_ASYN__ROUTING_FIRST+"*").noargs();
}
@Bean
public Binding BINDING_EAB_ASYN_TASK_QUEUE_SECOND(@Qualifier(EAB_ASYN_TASK_QUEUE_SECOND) Queue queue,
Exchange exchange){
return BindingBuilder.bind(queue).to(exchange).with(EAB_ASYN_ROUTING_SECOND+"*").noargs();
}
@Bean
public Binding BINDING_EAB_ASYN_TASK_QUEUE_THIRD(@Qualifier(EAB_ASYN_TASK_QUEUE_THIRD) Queue queue,
Exchange exchange){
return BindingBuilder.bind(queue).to(exchange).with(EAB_ASYN_ROUTING_THIRD+".*").noargs();
}
@Bean
public Binding BINDING_EAB_ASYN_TASK_QUEUE_FOURTH(@Qualifier(EAB_ASYN_TASK_QUEUE_FOURTH) Queue queue,
Exchange exchange){
return BindingBuilder.bind(queue).to(exchange).with(EAB_ASYN_ROUTING_FOURTH+"*").noargs();
}
@Bean
public Binding BINDING_EAB_ASYN_TASK_QUEUE_FIFTH(@Qualifier(EAB_ASYN_TASK_QUEUE_ZERO) Queue queue,
Exchange exchange){
return BindingBuilder.bind(queue).to(exchange).with(EAB_ASYN_ROUTING_ZERO+"*").noargs();
}
}
生产者:
private void sendMassageTopic(String enterpriseId, AsynTask<Map<String, Object>> at) {
if(StringUtils.isEmpty(enterpriseId) || null == at){
logger.error("发送消息参数不完整!拒绝此消息!");
return ;
}
//获取路由键
String routingKey;
String hashEnt = StringUtil.getEntIdIndex(enterpriseId,rabbitMqParam.getGroupSize());
if(this.checkPriorityEnterpriseId(enterpriseId)){
if (logger.isInfoEnabled()) {
logger.info(">>>>>>>>>>>>{} 企业发送数据变动优先队列 ", enterpriseId);
}
this.rabbitMq.convertAndSend(AsynTaskLogTopicConfig.EAB_ASYN_TASK_QUEUE_PRIORITY, JsonUtils.toJsonStr(at),
message -> {
message.getMessageProperties().setPriority(rabbitMqParam.getPriorityParameter());
return message;
});
}else{
if (logger.isInfoEnabled()) {
logger.info(">>>>>>>>>>>>{} 企业发送Topic ", enterpriseId);
}
if(EabConstants.EAB_ASYN__ROUTING_FIRST.equals(hashEnt)){
routingKey = AsynTaskLogTopicConfig.EAB_ASYN__ROUTING_FIRST+hashEnt;
}else if(EabConstants.EAB_ASYN_ROUTING_SECOND.equals(hashEnt)){
routingKey = AsynTaskLogTopicConfig.EAB_ASYN_ROUTING_SECOND+hashEnt;
}else if(EabConstants.EAB_ASYN_ROUTING_SECOND.equals(hashEnt)){
routingKey = AsynTaskLogTopicConfig.EAB_ASYN_ROUTING_THIRD+hashEnt;
}else if(EabConstants.EAB_ASYN_ROUTING_SECOND.equals(hashEnt)){
routingKey = AsynTaskLogTopicConfig.EAB_ASYN_ROUTING_FOURTH+hashEnt;
}else{
routingKey = AsynTaskLogTopicConfig.EAB_ASYN_ROUTING_ZERO+hashEnt;
}
this.rabbitMq.convertAndSend(AsynTaskLogTopicConfig.EAB_ASYN__TOPIC,routingKey,JsonUtils.toJsonStr(at));
}
}
消费者 :
package com.aspire.eab.batch.application.job.WorkAsynTask;
import com.aspire.eab.batch.application.job.IncreaseImportJob;
import com.aspire.eab.batch.framework.constant.EabBatchConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Component
public class WorkReceiver {
private final Logger logger = LoggerFactory.getLogger(this.getClass());
@Autowired
private IncreaseImportJob importJob;
@RabbitListener(queues = EabBatchConstants.EAB_ASYN_TASK_QUEUE_FIRST)
public void EAB_ASYN_TASK_QUEUE_FIRST(String message) {
if (logger.isInfoEnabled()) {
logger.info("FIRST 消息体mq请求参数:{}", message);
}
process(message);
}
@RabbitListener(queues = EabBatchConstants.EAB_ASYN_TASK_QUEUE_SECOND)
public void EAB_ASYN_TASK_QUEUE_SECOND(String message) {
if (logger.isInfoEnabled()) {
logger.info("SECOND 消息体mq请求参数:{}", message);
}
process(message);
}
@RabbitListener(queues = EabBatchConstants.EAB_ASYN_TASK_QUEUE_THIRD)
public void EAB_ASYN_TASK_QUEUE_THIRD(String message) {
if (logger.isInfoEnabled()) {
logger.info("THIRD 消息体mq请求参数:{}", message);
}
process(message);
}
@RabbitListener(queues = EabBatchConstants.EAB_ASYN_TASK_QUEUE_FOURTH)
public void EAB_ASYN_TASK_QUEUE_FOURTH(String message) {
if (logger.isInfoEnabled()) {
logger.info("FOURTH 消息体mq请求参数:{}", message);
}
process(message);
}
@RabbitListener(queues = EabBatchConstants.EAB_ASYN_TASK_QUEUE_ZERO)
public void EAB_ASYN_TASK_QUEUE_ZERO(String message) {
if (logger.isInfoEnabled()) {
logger.info("ZERO 消息体mq请求参数:{}", message);
}
process(message);
}
@RabbitListener(queues = EabBatchConstants.EAB_ASYN_TASK_QUEUE_PRIORITY)
public void EAB_ASYN_TASK_QUEUE_PRIORITY(String message) {
if (logger.isInfoEnabled()) {
logger.info("PRIORITY 消息体mq请求参数:{}", message);
}
process(message);
}
private void process(String message) {
this.importJob.process(message);
}
}
生产者项目mq配置
@Configuration
public class RabbitMqConfig {
@Autowired
private RabbitMqParam rabbitMqParam;
private static final Logger LOGGER = LoggerFactory.getLogger(RabbitMqConfig.class);
/**
* rabbitmq连接工厂
*
* @return ConnectionFactory
*/
@Bean
public ConnectionFactory connectionFactory() {
CachingConnectionFactory connectionFactory = new CachingConnectionFactory();
connectionFactory.setAddresses(rabbitMqParam.getAddresses());
connectionFactory.setUsername(rabbitMqParam.getUsername());
connectionFactory.setPassword(rabbitMqParam.getPassword());
connectionFactory.setPublisherConfirms(rabbitMqParam.getPublisherconfirms());
return connectionFactory;
}
/**
* rabbitAdmin代理类
*
* @return RabbitAdmin
*/
@Bean
public RabbitAdmin rabbitAdmin(
@Qualifier("connectionFactory") ConnectionFactory connectionFactory) {
return new RabbitAdmin(connectionFactory);
}
/**
* 创建rabbitTemplate 消息模板类
* prototype原型模式:每次获取Bean的时候会有一个新的实例
* 因为要设置回调类,所以应是prototype类型,如果是singleton类型,则回调类为最后一次设置
*
* @return RabbitTemplate
*/
@Bean
@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
public RabbitTemplate rabbitTemplate() {
RabbitTemplate rabbitTemplate = new RabbitTemplate(connectionFactory());
rabbitTemplate.setMessageConverter(new Jackson2JsonMessageConverter());
rabbitTemplate.setConfirmCallback(new ConfirmCallback() {
@Override
public void confirm(CorrelationData correlationData, boolean b, String s) {
if (!b) {
throw new RuntimeException("send error " + s);
}
}
});
return rabbitTemplate;
}
/**
* 监听工厂 - 全局
*
* @return SimpleRabbitListenerContainerFactory
*/
@Bean
public SimpleRabbitListenerContainerFactory rabbitListenerContainerFactory(
ConnectionFactory connectionFactory, ThreadPoolTaskExecutor threadPoolTaskExecutor) {
return this.getContainerFactory(connectionFactory, threadPoolTaskExecutor,
rabbitMqParam.getConcurrentConsumers(),
rabbitMqParam.getMaxConcurrentConsumers(),
rabbitMqParam.getPrefetch());
}
/**
* 监听工厂 - 需要单独引用的高并发消息消费
*
* @return SimpleRabbitListenerContainerFactory
*/
@Bean("highConcurContainerFactory")
public SimpleRabbitListenerContainerFactory pointTaskContainerFactory(
ConnectionFactory connectionFactory,ThreadPoolTaskExecutor threadPoolTaskExecutor) {
return this.getContainerFactory(connectionFactory, threadPoolTaskExecutor,
rabbitMqParam.getHighConcurConsumers(),
rabbitMqParam.getHighConcurMaxConsumers(),
rabbitMqParam.getHighConcurPrefetch());
}
/**
* 监听工厂 - 统一设置
*
* @return SimpleRabbitListenerContainerFactory
*/
private SimpleRabbitListenerContainerFactory getContainerFactory(ConnectionFactory connectionFactory,
ThreadPoolTaskExecutor threadPoolTaskExecutor,
Integer concurrentConsumers,
Integer maxConcurrentConsumers,
Integer prefetchCount){
SimpleRabbitListenerContainerFactory factory = new SimpleRabbitListenerContainerFactory();
factory.setConnectionFactory(connectionFactory);
factory.setMessageConverter(new Jackson2JsonMessageConverter());
//每个队列默认消费者数量
factory.setConcurrentConsumers(concurrentConsumers);
//每个队列最高消费者数量
// if (maxConcurrentConsumers > concurrentConsumers) {
factory.setMaxConcurrentConsumers(maxConcurrentConsumers);
// }
//单次消费拉取的消息个数
factory.setPrefetchCount(prefetchCount);
//指定执行容器
//factory.setTaskExecutor(threadPoolTaskExecutor);
factory.setTaskExecutor(this.consumersBrianThreadPool());
return factory;
}
@Bean("Consumers_TaskExecutor")
public ThreadPoolTaskExecutor consumersBrianThreadPool(){
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
//核心线程数
executor.setCorePoolSize(rabbitMqParam.getConsumersCorePoolSize());
//最大线程数
executor.setMaxPoolSize(rabbitMqParam.getConsumersMaxPoolSize());
// //队列中最大的数 默认值
// executor.setQueueCapacity(Integer.MAX_VALUE);
executor.setAllowCoreThreadTimeOut(true);
executor.setWaitForTasksToCompleteOnShutdown(true);
//县城名称前缀
executor.setThreadNamePrefix("consumersThreadPool_");
//对拒绝task的处理策略
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
//线程空闲后最大的存活时间 设置默认值
executor.setKeepAliveSeconds(60);
//初始化加载
executor.initialize();
if (LOGGER.isInfoEnabled()) {
LOGGER.info("核心线程数:{},最大线程数:{},线程空闲time :{}", executor.getCorePoolSize(),executor.getMaxPoolSize(),executor.getKeepAliveSeconds());
}
return executor;
}
}
其他代码就不贴了,都是写基础,另外 StringUtil.getEntIdIndex(enterpriseId,rabbitMqParam.getGroupSize());为hash 求模运算, 来入队的, 本来想搞个策略来完善这块, 时间有限。将就着吧。 。。