RocketMq源码解析之四(生产者消息发送)

ゞ浴缸里的玫瑰 2022-12-01 05:24 215阅读 0赞

### 文章目录 ###

*  消息发送
 *  消息启动流程
 *   *  获取mQClientFactory
 *  消息发送流程
 *   *  消息校验
     *  发送消息
     *  查找主题路由信息
     *  选择消息队列
     *   *  不启用故障延迟机制
         *  启用故障延迟机制
         *  更新失败条目
     *  同步发送消息
     *  异步发送消息
     *  批量发送消息

# 消息发送 #

RocketMQ 支持3 种消息发送方式：同步（sync） 、异步（async）、单向（o neway） 。  
同步： 发送者向MQ 执行发送消息API 时，同步等待， 直到消息服务器返回发送结果。  
异步： 发送者向MQ 执行发送消息API 时，指定消息发送成功后的回掉函数，然后调  
用消息发送API 后，立即返回，消息发送者线程不阻塞，直到运行结束，消息发送成功或  
失败的回调任务在一个新的线程中执行。  
单向：消息发送者向MQ 执行发送消息AP I 时，直接返回，不等待消息服务器的结果，  
也不注册回调函数，简单地说，就是只管发，不在乎消息是否成功存储在消息服务器上。

“单向”通信模式相对简单，一般用在发送心跳包场景下，无需关注其Response  
[具体看着篇官方介绍][Link 1]

# 消息启动流程 #

@Override
        public void start() throws MQClientException {
        
            //设置生产组, 生产者所属组，消息服务器在回查事务状态时会随机选择该组中任何一个生产者发起事务回查请求。
            this.setProducerGroup(withNamespace(this.producerGroup));
            this.defaultMQProducerImpl.start();
            if (null != traceDispatcher) {
        
                try {
        
                    traceDispatcher.start(this.getNamesrvAddr(), this.getAccessChannel());
                } catch (MQClientException e) {
        
                    log.warn("trace dispatcher start failed ", e);
                }
            }
        }

public void start() throws MQClientException {
        
            //是否启用MQClientInstance
            this.start(true);
        }
    
    public void start(final boolean startFactory) throws MQClientException {
        
            switch (this.serviceState) {
        
                //每次初始化状态CREATE_JUST
                case CREATE_JUST:
                    this.serviceState = ServiceState.START_FAILED;
                    //校验组为空,是否与默认组相同等
                    this.checkConfig();
    
                    //检查productGroup 是否符合要求
                    if (!this.defaultMQProducer.getProducerGroup().equals(MixAll.CLIENT_INNER_PRODUCER_GROUP)) {
        
                        //改变生产者的instanceName 为进程ID
                        this.defaultMQProducer.changeInstanceNameToPID();
                    }
    
                    //创建MQClientlnstance,同一个clientId只有一个实例
                    //MQClientlnstance 封装了RocketMQ 网络处理API ，是消息生产者（ Producer ）、消息消费者
                    //( Consumer ）与NameServer、Broker 打交道的网络通道。
    
                    //同一台服务器部署2个应用程序,且instance为默认值DEFAULT的话,clientId设置为进程id
                    //但是同一jvm中不同消费者和生产者在启动时获取到的MQClientlnstance都是同一个
                    this.mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(this.defaultMQProducer, rpcHook);
    
                    //向MQClientlnstance 注册，将当前生产者加入到MQClientlnstance 管理中，方便后续调用网络请求、进行心跳检测等
                    boolean registerOK = mQClientFactory.registerProducer(this.defaultMQProducer.getProducerGroup(), this);
                    if (!registerOK) {
        
                        this.serviceState = ServiceState.CREATE_JUST;
                        throw new MQClientException("The producer group[" + this.defaultMQProducer.getProducerGroup()
                            + "] has been created before, specify another name please." + FAQUrl.suggestTodo(FAQUrl.GROUP_NAME_DUPLICATE_URL),
                            null);
                    }
                    //将默认topic加入topic发布信息表中
                    this.topicPublishInfoTable.put(this.defaultMQProducer.getCreateTopicKey(), new TopicPublishInfo());
    
                    if (startFactory) {
        
                        //启动MQClientlnstance ，如果MQClientlnstance 已经启动，则本次启动不会真正执行
                        mQClientFactory.start();
                    }
    
                    log.info("the producer [{}] start OK. sendMessageWithVIPChannel={}", this.defaultMQProducer.getProducerGroup(),
                        this.defaultMQProducer.isSendMessageWithVIPChannel());
                    //更新状态
                    this.serviceState = ServiceState.RUNNING;
                    break;
                case RUNNING:
                case START_FAILED:
                case SHUTDOWN_ALREADY:
                    throw new MQClientException("The producer service state not OK, maybe started once, "
                        + this.serviceState
                        + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK),
                        null);
                default:
                    break;
            }
    
            this.mQClientFactory.sendHeartbeatToAllBrokerWithLock();
    
            this.timer.scheduleAtFixedRate(new TimerTask() {
        
                @Override
                public void run() {
        
                    try {
        
                        RequestFutureTable.scanExpiredRequest();
                    } catch (Throwable e) {
        
                        log.error("scan RequestFutureTable exception", e);
                    }
                }
            }, 1000 * 3, 1000);
        }
    
        private void checkConfig() throws MQClientException {
        
            //校验是否为空,以及字符串规则
            Validators.checkGroup(this.defaultMQProducer.getProducerGroup());
    
            //再次检查一边
            if (null == this.defaultMQProducer.getProducerGroup()) {
        
                throw new MQClientException("producerGroup is null", null);
            }
    
            //组为默认组,则异常
            if (this.defaultMQProducer.getProducerGroup().equals(MixAll.DEFAULT_PRODUCER_GROUP)) {
        
                throw new MQClientException("producerGroup can not equal " + MixAll.DEFAULT_PRODUCER_GROUP + ", please specify another one.",
                    null);
            }
        }

## 获取mQClientFactory ##

public MQClientInstance getOrCreateMQClientInstance(final ClientConfig clientConfig, RPCHook rpcHook) {
        
            //构建clientId
            String clientId = clientConfig.buildMQClientId();
            //缓存中通过clientId获取MQClient对象
            MQClientInstance instance = this.factoryTable.get(clientId);
            if (null == instance) {
        
                //构建MQclient实例对象
                instance =
                    new MQClientInstance(clientConfig.cloneClientConfig(),
                        this.factoryIndexGenerator.getAndIncrement(), clientId, rpcHook);
                //缓存MQClientInstance实例对象
                MQClientInstance prev = this.factoryTable.putIfAbsent(clientId, instance);
                //如果有与clientId绑定的实例对象,则返回
                if (prev != null) {
        
                    instance = prev;
                    log.warn("Returned Previous MQClientInstance for clientId:[{}]", clientId);
                } else {
        
                    log.info("Created new MQClientInstance for clientId:[{}]", clientId);
                }
            }
    
            return instance;
        }

# 消息发送流程 #

public SendResult send(
            Message msg) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
        
            //消息长度校验
            Validators.checkMessage(msg, this);
            msg.setTopic(withNamespace(msg.getTopic()));
            return this.defaultMQProducerImpl.send(msg);
        }

## 消息校验 ##

public static void checkMessage(Message msg, DefaultMQProducer defaultMQProducer)
            throws MQClientException {
        
            if (null == msg) {
        
                throw new MQClientException(ResponseCode.MESSAGE_ILLEGAL, "the message is null");
            }
            // topic
            //空校验,topic长度,以及topic命名校验
            Validators.checkTopic(msg.getTopic());
            //是否允许此topic发送 ,SCHEDULE_TOPIC_XXXX不允许
            Validators.isNotAllowedSendTopic(msg.getTopic());
    
            // body
            //body空校验,长度限制校验,
            if (null == msg.getBody()) {
        
                throw new MQClientException(ResponseCode.MESSAGE_ILLEGAL, "the message body is null");
            }
    
            if (0 == msg.getBody().length) {
        
                throw new MQClientException(ResponseCode.MESSAGE_ILLEGAL, "the message body length is zero");
            }
    
            if (msg.getBody().length > defaultMQProducer.getMaxMessageSize()) {
        
                throw new MQClientException(ResponseCode.MESSAGE_ILLEGAL,
                    "the message body size over max value, MAX: " + defaultMQProducer.getMaxMessageSize());
            }
        }

## 发送消息 ##

public SendResult send(
            Message msg) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
        
            //添加超时参数
            return send(msg, this.defaultMQProducer.getSendMsgTimeout());
        }
        public SendResult send(Message msg,
            long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
        
            //添加消息发送类型,默认同步,则回调函数为null
            return this.sendDefaultImpl(msg, CommunicationMode.SYNC, null, timeout);
        }
    
     private SendResult sendDefaultImpl(
            Message msg,
            final CommunicationMode communicationMode,
            final SendCallback sendCallback,
            final long timeout
        ) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
        
            //状态是否可运行
            this.makeSureStateOK();
            //空校验,以及长度校验
            Validators.checkMessage(msg, this.defaultMQProducer);
            final long invokeID = random.nextLong();
            long beginTimestampFirst = System.currentTimeMillis();
            long beginTimestampPrev = beginTimestampFirst;
            long endTimestamp = beginTimestampFirst;
            //查找主题路由信息
            TopicPublishInfo topicPublishInfo = this.tryToFindTopicPublishInfo(msg.getTopic());
            //该主题队列的消息队列不为空
            if (topicPublishInfo != null && topicPublishInfo.ok()) {
        
                boolean callTimeout = false;
                MessageQueue mq = null;
                Exception exception = null;
                SendResult sendResult = null;
                //同步超时重试次数默认2次+本身一次,3次.
                int timesTotal = communicationMode == CommunicationMode.SYNC ? 1 + this.defaultMQProducer.getRetryTimesWhenSendFailed() : 1;
                int times = 0;
                //记录三次选择消息队列的brokerName,用于出问题打印日志
                String[] brokersSent = new String[timesTotal];
                for (; times < timesTotal; times++) {
        
                    String lastBrokerName = null == mq ? null : mq.getBrokerName();
                    //选择消息队列
                    MessageQueue mqSelected = this.selectOneMessageQueue(topicPublishInfo, lastBrokerName);
                    if (mqSelected != null) {
        
                        mq = mqSelected;
                        brokersSent[times] = mq.getBrokerName();
                        try {
        
                            beginTimestampPrev = System.currentTimeMillis();
                            if (times > 0) {
        
                                //Reset topic with namespace during resend.
                                msg.setTopic(this.defaultMQProducer.withNamespace(msg.getTopic()));
                            }
                            long costTime = beginTimestampPrev - beginTimestampFirst;
                            if (timeout < costTime) {
        
                                //标记请求是否超时
                                callTimeout = true;
                                break;
                            }
    
                            //发送消息
                            sendResult = this.sendKernelImpl(msg, mq, communicationMode, sendCallback, topicPublishInfo, timeout - costTime);
                            endTimestamp = System.currentTimeMillis();
                            //更新失败条目。
                            this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false);
                            switch (communicationMode) {
        
                                case ASYNC:
                                    return null;
                                case ONEWAY:
                                    return null;
                                case SYNC:
                                    if (sendResult.getSendStatus() != SendStatus.SEND_OK) {
        
                                        if (this.defaultMQProducer.isRetryAnotherBrokerWhenNotStoreOK()) {
        
                                            continue;
                                        }
                                    }
    
                                    return sendResult;
                                default:
                                    break;
                            }
                        } catch (RemotingException e) {
        
                            endTimestamp = System.currentTimeMillis();
                            this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true);
                            log.warn(String.format("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq), e);
                            log.warn(msg.toString());
                            exception = e;
                            continue;
                        } catch (MQClientException e) {
        
                            endTimestamp = System.currentTimeMillis();
                            this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true);
                            log.warn(String.format("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq), e);
                            log.warn(msg.toString());
                            exception = e;
                            continue;
                        } catch (MQBrokerException e) {
        
                            endTimestamp = System.currentTimeMillis();
                            this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true);
                            log.warn(String.format("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq), e);
                            log.warn(msg.toString());
                            exception = e;
                            switch (e.getResponseCode()) {
        
                                case ResponseCode.TOPIC_NOT_EXIST:
                                case ResponseCode.SERVICE_NOT_AVAILABLE:
                                case ResponseCode.SYSTEM_ERROR:
                                case ResponseCode.NO_PERMISSION:
                                case ResponseCode.NO_BUYER_ID:
                                case ResponseCode.NOT_IN_CURRENT_UNIT:
                                    continue;
                                default:
                                    if (sendResult != null) {
        
                                        return sendResult;
                                    }
    
                                    throw e;
                            }
                        } catch (InterruptedException e) {
        
                            endTimestamp = System.currentTimeMillis();
                            this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false);
                            log.warn(String.format("sendKernelImpl exception, throw exception, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq), e);
                            log.warn(msg.toString());
    
                            log.warn("sendKernelImpl exception", e);
                            log.warn(msg.toString());
                            throw e;
                        }
                    } else {
        
                        break;
                    }
                }
    
                if (sendResult != null) {
        
                    return sendResult;
                }
    
                String info = String.format("Send [%d] times, still failed, cost [%d]ms, Topic: %s, BrokersSent: %s",
                    times,
                    System.currentTimeMillis() - beginTimestampFirst,
                    msg.getTopic(),
                    Arrays.toString(brokersSent));
    
                info += FAQUrl.suggestTodo(FAQUrl.SEND_MSG_FAILED);
    
                MQClientException mqClientException = new MQClientException(info, exception);
                if (callTimeout) {
        
                    throw new RemotingTooMuchRequestException("sendDefaultImpl call timeout");
                }
    
                if (exception instanceof MQBrokerException) {
        
                    mqClientException.setResponseCode(((MQBrokerException) exception).getResponseCode());
                } else if (exception instanceof RemotingConnectException) {
        
                    mqClientException.setResponseCode(ClientErrorCode.CONNECT_BROKER_EXCEPTION);
                } else if (exception instanceof RemotingTimeoutException) {
        
                    mqClientException.setResponseCode(ClientErrorCode.ACCESS_BROKER_TIMEOUT);
                } else if (exception instanceof MQClientException) {
        
                    mqClientException.setResponseCode(ClientErrorCode.BROKER_NOT_EXIST_EXCEPTION);
                }
    
                throw mqClientException;
            }
    
            validateNameServerSetting();
    
            throw new MQClientException("No route info of this topic: " + msg.getTopic() + FAQUrl.suggestTodo(FAQUrl.NO_TOPIC_ROUTE_INFO),
                null).setResponseCode(ClientErrorCode.NOT_FOUND_TOPIC_EXCEPTION);
        }
    
        private void makeSureStateOK() throws MQClientException {
        
            if (this.serviceState != ServiceState.RUNNING) {
        
                throw new MQClientException("The producer service state not OK, "
                    + this.serviceState
                    + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK),
                    null);
            }
        }

## 查找主题路由信息 ##

首先缓存中获取  
获取不到根据topic从远端name获取  
获取不到在根据默认topic TBW102获取  
如果BrokerConfig\#autoCreateTopicEnable 为true 时，NameServer 将返回路由信息  
如果autoCreateTopicEnable为false 将抛出无法找到topic 路由异常

//查找主题路由信息
            private TopicPublishInfo tryToFindTopicPublishInfo(final String topic) {
        
                //缓存中获取topic路由信息
                TopicPublishInfo topicPublishInfo = this.topicPublishInfoTable.get(topic);
                //如果缓存没路由信息,或没包含消息队列
                if (null == topicPublishInfo || !topicPublishInfo.ok()) {
        
                    //新增默认topic
                    this.topicPublishInfoTable.putIfAbsent(topic, new TopicPublishInfo());
                    //向nameServer查询topic路由信息
                    this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic);
                    //再次缓存获取
                    topicPublishInfo = this.topicPublishInfoTable.get(topic);
                }
                //如果缓存有路由信息,或包含消息队列
                if (topicPublishInfo.isHaveTopicRouterInfo() || topicPublishInfo.ok()) {
        
                    //返回路由信息
                    return topicPublishInfo;
                } else {
        
                /*
                用默认主题DefaultMQProducerlmpl#createTopicKey 去查询，
                如果BrokerConfig#autoCreateTopicEnable 为true 时，NameServer 将返回路由信息，如果
                autoCreateTopicEnable为false 将抛出无法找到topic 路由异常
                 */
                    this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic, true, this.defaultMQProducer);
                    topicPublishInfo = this.topicPublishInfoTable.get(topic);
                    return topicPublishInfo;
                }
        }

public boolean updateTopicRouteInfoFromNameServer(final String topic, boolean isDefault,
            DefaultMQProducer defaultMQProducer) {
        
            try {
        
                if (this.lockNamesrv.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) {
        
                    try {
        
                        TopicRouteData topicRouteData;
                        //如果用默认主题DefaultMQProducerlmpl#createTopicKey 去查询，
                        if (isDefault && defaultMQProducer != null) {
        
                            //查询路由信息
                            topicRouteData = this.mQClientAPIImpl.getDefaultTopicRouteInfoFromNameServer(defaultMQProducer.getCreateTopicKey(),
                                1000 * 3);
                            //如果查询到路由信息
                            if (topicRouteData != null) {
        
                                //替换路由信息中读写队列个数为消息生产者默认的队列个数
                                for (QueueData data : topicRouteData.getQueueDatas()) {
        
                                    int queueNums = Math.min(defaultMQProducer.getDefaultTopicQueueNums(), data.getReadQueueNums());
                                    data.setReadQueueNums(queueNums);
                                    data.setWriteQueueNums(queueNums);
                                }
                            }
                        } else {
        
                            //如果不使用默认主题查
                            topicRouteData = this.mQClientAPIImpl.getTopicRouteInfoFromNameServer(topic, 1000 * 3);
                        }
                        //如果未找到路由信息，则直接返回false
                        //如果找到路由信息,继续往下执行
                        if (topicRouteData != null) {
        
                            TopicRouteData old = this.topicRouteTable.get(topic);
                            //与本地缓存中的路由信息进行对比
                            boolean changed = topicRouteDataIsChange(old, topicRouteData);
                            if (!changed) {
        
                                changed = this.isNeedUpdateTopicRouteInfo(topic);
                            } else {
        
                                log.info("the topic[{}] route info changed, old[{}] ,new[{}]", topic, old, topicRouteData);
                            }
                            //如果未发生变化，则直接返回false
                            //否则继续往下执行
                            if (changed) {
        
                                TopicRouteData cloneTopicRouteData = topicRouteData.cloneTopicRouteData();
                                //更新MQClientlnstance Broker 地址缓存表。
                                for (BrokerData bd : topicRouteData.getBrokerDatas()) {
        
                                    this.brokerAddrTable.put(bd.getBrokerName(), bd.getBrokerAddrs());
                                }
    
                                // Update Pub info
                                {
        
                                    //根据topicRouteData 中的List<QueueData> 转换成TopicPublishInfo的List<MessageQueue>列表
                                    TopicPublishInfo publishInfo = topicRouteData2TopicPublishInfo(topic, topicRouteData);
                                    //标记有主题路由信息
                                    publishInfo.setHaveTopicRouterInfo(true);
    
                                    Iterator<Entry<String, MQProducerInner>> it = this.producerTable.entrySet().iterator();
                                    while (it.hasNext()) {
        
                                        Entry<String, MQProducerInner> entry = it.next();
                                        MQProducerInner impl = entry.getValue();
                                        if (impl != null) {
        
                                            //更新该MQClientInstance管辖的所有消息发送关于topic 的路由信息。
                                            impl.updateTopicPublishInfo(topic, publishInfo);
                                        }
                                    }
                                }
    
                                // Update sub info
                                {
        
                                    Set<MessageQueue> subscribeInfo = topicRouteData2TopicSubscribeInfo(topic, topicRouteData);
                                    Iterator<Entry<String, MQConsumerInner>> it = this.consumerTable.entrySet().iterator();
                                    while (it.hasNext()) {
        
                                        Entry<String, MQConsumerInner> entry = it.next();
                                        MQConsumerInner impl = entry.getValue();
                                        if (impl != null) {
        
                                            impl.updateTopicSubscribeInfo(topic, subscribeInfo);
                                        }
                                    }
                                }
                                log.info("topicRouteTable.put. Topic = {}, TopicRouteData[{}]", topic, cloneTopicRouteData);
                                this.topicRouteTable.put(topic, cloneTopicRouteData);
                                return true;
                            }
                        } else {
        
                            log.warn("updateTopicRouteInfoFromNameServer, getTopicRouteInfoFromNameServer return null, Topic: {}", topic);
                        }
                    } catch (MQClientException e) {
        
                        if (!topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
        
                            log.warn("updateTopicRouteInfoFromNameServer Exception", e);
                        }
                    } catch (RemotingException e) {
        
                        log.error("updateTopicRouteInfoFromNameServer Exception", e);
                        throw new IllegalStateException(e);
                    } finally {
        
                        this.lockNamesrv.unlock();
                    }
                } else {
        
                    log.warn("updateTopicRouteInfoFromNameServer tryLock timeout {}ms", LOCK_TIMEOUT_MILLIS);
                }
            } catch (InterruptedException e) {
        
                log.warn("updateTopicRouteInfoFromNameServer Exception", e);
            }
    
            return false;
        }

## 选择消息队列 ##

选择消息队列有两种方式。  
1 ) sendLatencyFaultEnable=false ，默认不启用Broker 故障延迟机制。  
2 ) sendLatencyFaultEnable=true ，启用Broker 故障延迟机制。

public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName) {
        
            return this.mqFaultStrategy.selectOneMessageQueue(tpInfo, lastBrokerName);
        }

public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName) {
        
            if (this.sendLatencyFaultEnable) {
        
            //启用Broker 故障延迟机制
                try {
        
                    //算出下一个队列索引
                    int index = tpInfo.getSendWhichQueue().getAndIncrement();
                    for (int i = 0; i < tpInfo.getMessageQueueList().size(); i++) {
        
                        int pos = Math.abs(index++) % tpInfo.getMessageQueueList().size();
                        if (pos < 0)
                            pos = 0;
                        MessageQueue mq = tpInfo.getMessageQueueList().get(pos);
                        //队列是否可用
                        if (latencyFaultTolerance.isAvailable(mq.getBrokerName())) {
        
                            //说明超过了规避时间
                            if (null == lastBrokerName || mq.getBrokerName().equals(lastBrokerName))
                                return mq;
                        }
                    }
    
                    //尝试从规避的Broker 中选择一个可用的Broker
                    final String notBestBroker = latencyFaultTolerance.pickOneAtLeast();
                    //根据brokerName获取队列中可写队列数量
                    int writeQueueNums = tpInfo.getQueueIdByBroker(notBestBroker);
                    //说明有队列可以选择
                    if (writeQueueNums > 0) {
        
                        //获取下一个索引队列
                        final MessageQueue mq = tpInfo.selectOneMessageQueue();
                        if (notBestBroker != null) {
        
                            mq.setBrokerName(notBestBroker);
                            //设置为下一个队列索引
                            mq.setQueueId(tpInfo.getSendWhichQueue().getAndIncrement() % writeQueueNums);
                        }
                        return mq;
                    } else {
        
                        //没队列可以选择,则从故障项移除
                        latencyFaultTolerance.remove(notBestBroker);
                    }
                } catch (Exception e) {
        
                    log.error("Error occurred when selecting message queue", e);
                }
                //获取下一个索引队列
                return tpInfo.selectOneMessageQueue();
            }
            //不启用Broker 故障延迟机制
            return tpInfo.selectOneMessageQueue(lastBrokerName);
        }

### 不启用故障延迟机制 ###

该算法在一次消息发送过程中能成功规避故障的B roker，但如果Bro ker 宕机，由于路
    由算法中的消息队列是按Broker 排序的，如果上一次根据路由算法选择的是宕机的Broker
    的第一个队列，那么随后的下次选择的是宕机Broker 的第二个队列，消息发送很有可能会
    失败，再次引发重试，带来不必要的性能损耗

public MessageQueue selectOneMessageQueue(final String lastBrokerName) {
        
            //说明第一次执行
            if (lastBrokerName == null) {
        
                return selectOneMessageQueue();
            } else {
        
                int index = this.sendWhichQueue.getAndIncrement();
                for (int i = 0; i < this.messageQueueList.size(); i++) {
        
                    int pos = Math.abs(index++) % this.messageQueueList.size();
                    if (pos < 0)
                        pos = 0;
                    MessageQueue mq = this.messageQueueList.get(pos);
                    //规避上次消息发送失败的broker
                    if (!mq.getBrokerName().equals(lastBrokerName)) {
        
                        return mq;
                    }
                }
                return selectOneMessageQueue();
            }
        }

### 启用故障延迟机制 ###

@Override
        public boolean isAvailable(final String name) {
        
            final FaultItem faultItem = this.faultItemTable.get(name);
            if (faultItem != null) {
        
                return faultItem.isAvailable();
            }
            return true;
        }
    
       public boolean isAvailable() {
        
                //当前时间是否超过故障规避终止时间
                return (System.currentTimeMillis() - startTimestamp) >= 0;
            }

@Override
        public String pickOneAtLeast() {
        
            //获取所有故障项
            final Enumeration<FaultItem> elements = this.faultItemTable.elements();
            List<FaultItem> tmpList = new LinkedList<FaultItem>();
            while (elements.hasMoreElements()) {
        
                final FaultItem faultItem = elements.nextElement();
                tmpList.add(faultItem);
            }
    
            if (!tmpList.isEmpty()) {
        
                Collections.shuffle(tmpList);
    
                Collections.sort(tmpList);
    
                //从故障项前面一半获取元素
                final int half = tmpList.size() / 2;
                if (half <= 0) {
        
                    return tmpList.get(0).getName();
                } else {
        
                    final int i = this.whichItemWorst.getAndIncrement() % half;
                    return tmpList.get(i).getName();
                }
            }
    
            return null;
        }

### 更新失败条目 ###

org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl\#sendDefaultImpl

/**
         *
         * @param brokerName
         * @param currentLatency   本次消息发送延迟时间currentLatency 。
         * @param isolation 是否隔离，该参数的含义如果为true ，则使用默认时长30s 来
         * 计算Broker 故障规避时长，如果为false ， 则使用本次消息发送延迟时间来计算Broker 故障规避时长。
         */
        public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation) {
        
            this.mqFaultStrategy.updateFaultItem(brokerName, currentLatency, isolation);
        }
    
    
        public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation) {
        
            if (this.sendLatencyFaultEnable) {
        
                //如果isolation 为true ，则使用30s 作为computeNotAvailableDuration 方法的参数；如
                //果isolation 为false ，则使用本次消息发送时延作为computeNotAvailableDuration 方法的参数，
                //获取需要规避的时长
                long duration = computeNotAvailableDuration(isolation ? 30000 : currentLatency);
                this.latencyFaultTolerance.updateFaultItem(brokerName, currentLatency, duration);
            }
        }

/*
        latency Max ，根据currentLatency 本次消息发送延迟，从latencyMax 尾部向前找到
        第一个比currentLatency 小的索引index，如果没有找到，返回0 。然后根据这个索引从
        notAvailableDuration 数组中取出对应的时间，在这个时长内， Broker 将设置为不可用。
         */
        private long[] latencyMax = {
        50L, 100L, 550L, 1000L, 2000L, 3000L, 15000L};
        private long[] notAvailableDuration = {
        0L, 0L, 30000L, 60000L, 120000L, 180000L, 600000L};
    
    
     private long computeNotAvailableDuration(final long currentLatency) {
        
            for (int i = latencyMax.length - 1; i >= 0; i--) {
        
                if (currentLatency >= latencyMax[i])
                    return this.notAvailableDuration[i];
            }
    
            return 0;
        }

@Override
        public void updateFaultItem(final String name, final long currentLatency, final long notAvailableDuration) {
        
            FaultItem old = this.faultItemTable.get(name);
            if (null == old) {
        
                final FaultItem faultItem = new FaultItem(name);
                faultItem.setCurrentLatency(currentLatency);
                //算出终止规避时间
                faultItem.setStartTimestamp(System.currentTimeMillis() + notAvailableDuration);
    
                old = this.faultItemTable.putIfAbsent(name, faultItem);
                if (old != null) {
        
                    old.setCurrentLatency(currentLatency);
                    old.setStartTimestamp(System.currentTimeMillis() + notAvailableDuration);
                }
            } else {
        
                old.setCurrentLatency(currentLatency);
                old.setStartTimestamp(System.currentTimeMillis() + notAvailableDuration);
            }
        }

## 同步发送消息 ##

private SendResult sendKernelImpl(final Message msg,
            final MessageQueue mq,
            final CommunicationMode communicationMode,
            final SendCallback sendCallback,
            final TopicPublishInfo topicPublishInfo,
            final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
        
            long beginStartTime = System.currentTimeMillis();
            //缓存中获取master brokerAddr
            String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName());
            if (null == brokerAddr) {
        
                //查找主题路由信息
                //从NameServer 主动更新一下topic的路由信息。
                tryToFindTopicPublishInfo(mq.getTopic());
                //再次缓存获取
                brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName());
            }
    
            SendMessageContext context = null;
            //依旧没有,则抛异常
            if (brokerAddr != null) {
        
                brokerAddr = MixAll.brokerVIPChannel(this.defaultMQProducer.isSendMessageWithVIPChannel(), brokerAddr);
    
                byte[] prevBody = msg.getBody();
                try {
        
                    //for MessageBatch,ID has been set in the generating process
                    //为消息分配全局唯一ID,发送方幂等就是这里保持的
                    if (!(msg instanceof MessageBatch)) {
        
                        MessageClientIDSetter.setUniqID(msg);
                    }
    
                    boolean topicWithNamespace = false;
                    if (null != this.mQClientFactory.getClientConfig().getNamespace()) {
        
                        msg.setInstanceId(this.mQClientFactory.getClientConfig().getNamespace());
                        topicWithNamespace = true;
                    }
    
                    int sysFlag = 0;
                    boolean msgBodyCompressed = false;
                    //压缩消息体
                    if (this.tryToCompressMessage(msg)) {
        
                        //标记COMPRESSED_FLAG
                        sysFlag |= MessageSysFlag.COMPRESSED_FLAG;
                        msgBodyCompressed = true;
                    }
    
                    //如果是事务消息
                    final String tranMsg = msg.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED);
                    if (tranMsg != null && Boolean.parseBoolean(tranMsg)) {
        
                        //标记TRANSACTION_PREPARED_TYPE
                        sysFlag |= MessageSysFlag.TRANSACTION_PREPARED_TYPE;
                    }
    
                    if (hasCheckForbiddenHook()) {
        
                        CheckForbiddenContext checkForbiddenContext = new CheckForbiddenContext();
                        checkForbiddenContext.setNameSrvAddr(this.defaultMQProducer.getNamesrvAddr());
                        checkForbiddenContext.setGroup(this.defaultMQProducer.getProducerGroup());
                        checkForbiddenContext.setCommunicationMode(communicationMode);
                        checkForbiddenContext.setBrokerAddr(brokerAddr);
                        checkForbiddenContext.setMessage(msg);
                        checkForbiddenContext.setMq(mq);
                        checkForbiddenContext.setUnitMode(this.isUnitMode());
                        this.executeCheckForbiddenHook(checkForbiddenContext);
                    }
    
                    //如果注册了消息发送钩子函数,通过registerSendMessageHook注册
                    if (this.hasSendMessageHook()) {
        
                        context = new SendMessageContext();
                        context.setProducer(this);
                        context.setProducerGroup(this.defaultMQProducer.getProducerGroup());
                        context.setCommunicationMode(communicationMode);
                        context.setBornHost(this.defaultMQProducer.getClientIP());
                        context.setBrokerAddr(brokerAddr);
                        context.setMessage(msg);
                        context.setMq(mq);
                        context.setNamespace(this.defaultMQProducer.getNamespace());
                        String isTrans = msg.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED);
                        if (isTrans != null && isTrans.equals("true")) {
        
                            context.setMsgType(MessageType.Trans_Msg_Half);
                        }
    
                        if (msg.getProperty("__STARTDELIVERTIME") != null || msg.getProperty(MessageConst.PROPERTY_DELAY_TIME_LEVEL) != null) {
        
                            context.setMsgType(MessageType.Delay_Msg);
                        }
                        //执行消息发送之前的增强逻辑
                        this.executeSendMessageHookBefore(context);
                    }
    
                    SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();
                    requestHeader.setProducerGroup(this.defaultMQProducer.getProducerGroup());
                    requestHeader.setTopic(msg.getTopic());
                    requestHeader.setDefaultTopic(this.defaultMQProducer.getCreateTopicKey());
                    requestHeader.setDefaultTopicQueueNums(this.defaultMQProducer.getDefaultTopicQueueNums());
                    requestHeader.setQueueId(mq.getQueueId());
                    requestHeader.setSysFlag(sysFlag);
                    //消息发送时间
                    requestHeader.setBornTimestamp(System.currentTimeMillis());
                    //消息标记,rocket不处理,仅供系统使用
                    requestHeader.setFlag(msg.getFlag());
                    //消息扩展属性
                    requestHeader.setProperties(MessageDecoder.messageProperties2String(msg.getProperties()));
                    requestHeader.setReconsumeTimes(0);
                    requestHeader.setUnitMode(this.isUnitMode());
                    //是否批量消息
                    requestHeader.setBatch(msg instanceof MessageBatch);
                    if (requestHeader.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
        
                        String reconsumeTimes = MessageAccessor.getReconsumeTime(msg);
                        if (reconsumeTimes != null) {
        
                            requestHeader.setReconsumeTimes(Integer.valueOf(reconsumeTimes));
                            MessageAccessor.clearProperty(msg, MessageConst.PROPERTY_RECONSUME_TIME);
                        }
    
                        String maxReconsumeTimes = MessageAccessor.getMaxReconsumeTimes(msg);
                        if (maxReconsumeTimes != null) {
        
                            requestHeader.setMaxReconsumeTimes(Integer.valueOf(maxReconsumeTimes));
                            MessageAccessor.clearProperty(msg, MessageConst.PROPERTY_MAX_RECONSUME_TIMES);
                        }
                    }
    
                    SendResult sendResult = null;
                    switch (communicationMode) {
        
                        //异步发送
                        case ASYNC:
                            Message tmpMessage = msg;
                            boolean messageCloned = false;
                            if (msgBodyCompressed) {
        
                                //If msg body was compressed, msgbody should be reset using prevBody.
                                //Clone new message using commpressed message body and recover origin massage.
                                //Fix bug:https://github.com/apache/rocketmq-externals/issues/66
                                tmpMessage = MessageAccessor.cloneMessage(msg);
                                messageCloned = true;
                                msg.setBody(prevBody);
                            }
    
                            if (topicWithNamespace) {
        
                                if (!messageCloned) {
        
                                    tmpMessage = MessageAccessor.cloneMessage(msg);
                                    messageCloned = true;
                                }
                                msg.setTopic(NamespaceUtil.withoutNamespace(msg.getTopic(), this.defaultMQProducer.getNamespace()));
                            }
    
                            //超时处理
                            long costTimeAsync = System.currentTimeMillis() - beginStartTime;
                            if (timeout < costTimeAsync) {
        
                                throw new RemotingTooMuchRequestException("sendKernelImpl call timeout");
                            }
                            sendResult = this.mQClientFactory.getMQClientAPIImpl().sendMessage(
                                brokerAddr,
                                mq.getBrokerName(),
                                tmpMessage,
                                requestHeader,
                                timeout - costTimeAsync,
                                communicationMode,
                                sendCallback,
                                topicPublishInfo,
                                this.mQClientFactory,
                                this.defaultMQProducer.getRetryTimesWhenSendAsyncFailed(),
                                context,
                                this);
                            break;
                            //单向发送
                        case ONEWAY:
                            //同步发送
                        case SYNC:
                            long costTimeSync = System.currentTimeMillis() - beginStartTime;
                            //执行到这里若超时,则异常
                            if (timeout < costTimeSync) {
        
                                throw new RemotingTooMuchRequestException("sendKernelImpl call timeout");
                            }
                            sendResult = this.mQClientFactory.getMQClientAPIImpl().sendMessage(
                                brokerAddr,
                                mq.getBrokerName(),
                                msg,
                                requestHeader,
                                timeout - costTimeSync,
                                communicationMode,
                                context,
                                this);
                            break;
                        default:
                            assert false;
                            break;
                    }
    
                    //如果有钩子函数,执行钩子函数后置方法
                    if (this.hasSendMessageHook()) {
        
                        context.setSendResult(sendResult);
                        this.executeSendMessageHookAfter(context);
                    }
    
                    return sendResult;
                } catch (RemotingException e) {
        
                    if (this.hasSendMessageHook()) {
        
                        context.setException(e);
                        this.executeSendMessageHookAfter(context);
                    }
                    throw e;
                } catch (MQBrokerException e) {
        
                    if (this.hasSendMessageHook()) {
        
                        context.setException(e);
                        this.executeSendMessageHookAfter(context);
                    }
                    throw e;
                } catch (InterruptedException e) {
        
                    if (this.hasSendMessageHook()) {
        
                        context.setException(e);
                        this.executeSendMessageHookAfter(context);
                    }
                    throw e;
                } finally {
        
                    msg.setBody(prevBody);
                    msg.setTopic(NamespaceUtil.withoutNamespace(msg.getTopic(), this.defaultMQProducer.getNamespace()));
                }
            }
    
            throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null);
        }

## 异步发送消息 ##

private void sendMessageAsync(
            final String addr,
            final String brokerName,
            final Message msg,
            final long timeoutMillis,
            final RemotingCommand request,
            final SendCallback sendCallback,
            final TopicPublishInfo topicPublishInfo,
            final MQClientInstance instance,
            final int retryTimesWhenSendFailed,
            final AtomicInteger times,
            final SendMessageContext context,
            final DefaultMQProducerImpl producer
        ) throws InterruptedException, RemotingException {
        
            final long beginStartTime = System.currentTimeMillis();
            //异步发送消息
            this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() {
        
                @Override
                public void operationComplete(ResponseFuture responseFuture) {
        
                    long cost = System.currentTimeMillis() - beginStartTime;
                    RemotingCommand response = responseFuture.getResponseCommand();
                    if (null == sendCallback && response != null) {
        
    
                        try {
        
                            //获取发送结果
                            SendResult sendResult = MQClientAPIImpl.this.processSendResponse(brokerName, msg, response);
                            if (context != null && sendResult != null) {
        
                                context.setSendResult(sendResult);
                                //执行钩子函数
                                context.getProducer().executeSendMessageHookAfter(context);
                            }
                        } catch (Throwable e) {
        
                        }
    
                        //修改异常条目
                        producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false);
                        return;
                    }
    
                    if (response != null) {
        
                        try {
        
                            SendResult sendResult = MQClientAPIImpl.this.processSendResponse(brokerName, msg, response);
                            assert sendResult != null;
                            if (context != null) {
        
                                context.setSendResult(sendResult);
                                context.getProducer().executeSendMessageHookAfter(context);
                            }
    
                            try {
        
                                sendCallback.onSuccess(sendResult);
                            } catch (Throwable e) {
        
                            }
    
                            producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false);
                        } catch (Exception e) {
        
                            producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true);
                            onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance,
                                retryTimesWhenSendFailed, times, e, context, false, producer);
                        }
                    } else {
        
                        producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true);
                        if (!responseFuture.isSendRequestOK()) {
        
                            MQClientException ex = new MQClientException("send request failed", responseFuture.getCause());
                            onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance,
                                retryTimesWhenSendFailed, times, ex, context, true, producer);
                        } else if (responseFuture.isTimeout()) {
        
                            MQClientException ex = new MQClientException("wait response timeout " + responseFuture.getTimeoutMillis() + "ms",
                                responseFuture.getCause());
                            onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance,
                                retryTimesWhenSendFailed, times, ex, context, true, producer);
                        } else {
        
                            MQClientException ex = new MQClientException("unknow reseaon", responseFuture.getCause());
                            onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance,
                                retryTimesWhenSendFailed, times, ex, context, true, producer);
                        }
                    }
                }
            });
        }

**NettyRemotingClient**

@Override
        public void invokeAsync(String addr, RemotingCommand request, long timeoutMillis, InvokeCallback invokeCallback)
            throws InterruptedException, RemotingConnectException, RemotingTooMuchRequestException, RemotingTimeoutException,
            RemotingSendRequestException {
        
            long beginStartTime = System.currentTimeMillis();
            //根据地址获取通道,没有则创建
            final Channel channel = this.getAndCreateChannel(addr);
            if (channel != null && channel.isActive()) {
        
                try {
        
                    //rpc前置钩子
                    doBeforeRpcHooks(addr, request);
                    long costTime = System.currentTimeMillis() - beginStartTime;
                    //超时处理
                    if (timeoutMillis < costTime) {
        
                        throw new RemotingTooMuchRequestException("invokeAsync call timeout");
                    }
                    //异步调用
                    this.invokeAsyncImpl(channel, request, timeoutMillis - costTime, invokeCallback);
                } catch (RemotingSendRequestException e) {
        
                    log.warn("invokeAsync: send request exception, so close the channel[{}]", addr);
                    this.closeChannel(addr, channel);
                    throw e;
                }
            } else {
        
                this.closeChannel(addr, channel);
                throw new RemotingConnectException(addr);
            }
        }

**NettyRemotingAbstract**

public void invokeAsyncImpl(final Channel channel, final RemotingCommand request, final long timeoutMillis,
            final InvokeCallback invokeCallback)
            throws InterruptedException, RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException {
        
            long beginStartTime = System.currentTimeMillis();
            final int opaque = request.getOpaque();
            //信号量限制并发请求
            boolean acquired = this.semaphoreAsync.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS);
            if (acquired) {
        
                final SemaphoreReleaseOnlyOnce once = new SemaphoreReleaseOnlyOnce(this.semaphoreAsync);
                long costTime = System.currentTimeMillis() - beginStartTime;
                if (timeoutMillis < costTime) {
        
                    //超时,则释放
                    once.release();
                    throw new RemotingTimeoutException("invokeAsyncImpl call timeout");
                }
    
                final ResponseFuture responseFuture = new ResponseFuture(channel, opaque, timeoutMillis - costTime, invokeCallback, once);
                //存放请求
                this.responseTable.put(opaque, responseFuture);
                try {
        
                    //发送请求
                    channel.writeAndFlush(request).addListener(new ChannelFutureListener() {
        
                        @Override
                        public void operationComplete(ChannelFuture f) throws Exception {
        
                            if (f.isSuccess()) {
        
                                responseFuture.setSendRequestOK(true);
                                return;
                            }
                            //请求失败回调
                            requestFail(opaque);
                            log.warn("send a request command to channel <{}> failed.", RemotingHelper.parseChannelRemoteAddr(channel));
                        }
                    });
                } catch (Exception e) {
        
                    responseFuture.release();
                    log.warn("send a request command to channel <" + RemotingHelper.parseChannelRemoteAddr(channel) + "> Exception", e);
                    throw new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e);
                }
            } else {
        
                if (timeoutMillis <= 0) {
        
                    throw new RemotingTooMuchRequestException("invokeAsyncImpl invoke too fast");
                } else {
        
                    String info =
                        String.format("invokeAsyncImpl tryAcquire semaphore timeout, %dms, waiting thread nums: %d semaphoreAsyncValue: %d",
                            timeoutMillis,
                            this.semaphoreAsync.getQueueLength(),
                            this.semaphoreAsync.availablePermits()
                        );
                    log.warn(info);
                    throw new RemotingTimeoutException(info);
                }
            }
        }

异步回调执行流程:

class NettyClientHandler extends SimpleChannelInboundHandler<RemotingCommand> {
        
    
            @Override
            //处理消息接收
            protected void channelRead0(ChannelHandlerContext ctx, RemotingCommand msg) throws Exception {
        
                processMessageReceived(ctx, msg);
            }
        }

**NettyRemotingAbstract**

public void processMessageReceived(ChannelHandlerContext ctx, RemotingCommand msg) throws Exception {
        
            final RemotingCommand cmd = msg;
            if (cmd != null) {
        
                switch (cmd.getType()) {
        
                    case REQUEST_COMMAND:
                        processRequestCommand(ctx, cmd);
                        break;
                    case RESPONSE_COMMAND:
                        //处理响应
                        processResponseCommand(ctx, cmd);
                        break;
                    default:
                        break;
                }
            }
        }

**NettyRemotingAbstract**

public void processResponseCommand(ChannelHandlerContext ctx, RemotingCommand cmd) {
        
            //获取请求
            final int opaque = cmd.getOpaque();
            final ResponseFuture responseFuture = responseTable.get(opaque);
            if (responseFuture != null) {
        
                responseFuture.setResponseCommand(cmd);
    
                //删除请求记录
                responseTable.remove(opaque);
    
                if (responseFuture.getInvokeCallback() != null) {
        
                    //执行回调
                    executeInvokeCallback(responseFuture);
                } else {
        
                    //没有回调函数
                    //放入cmd
                    responseFuture.putResponse(cmd);
                    //释放xinhaoli
                    responseFuture.release();
                }
            } else {
        
                log.warn("receive response, but not matched any request, " + RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
                log.warn(cmd.toString());
            }
        }

**NettyRemotingAbstract**

private void executeInvokeCallback(final ResponseFuture responseFuture) {
        
            boolean runInThisThread = false;
            //获取回调函数线程池,默认是publicExecutor线程池
            ExecutorService executor = this.getCallbackExecutor();
            if (executor != null) {
        
                try {
        
                    executor.submit(new Runnable() {
        
                        @Override
                        public void run() {
        
                            try {
        
                                //执行回调
                                responseFuture.executeInvokeCallback();
                            } catch (Throwable e) {
        
                                log.warn("execute callback in executor exception, and callback throw", e);
                            } finally {
        
                                responseFuture.release();
                            }
                        }
                    });
                } catch (Exception e) {
        
                    runInThisThread = true;
                    log.warn("execute callback in executor exception, maybe executor busy", e);
                }
            } else {
        
                runInThisThread = true;
            }
    
            //执行失败，或者找不到线程池
            if (runInThisThread) {
        
                try {
        
                    //当前线程执行
                    responseFuture.executeInvokeCallback();
                } catch (Throwable e) {
        
                    log.warn("executeInvokeCallback Exception", e);
                } finally {
        
                    //释放信号量
                    responseFuture.release();
                }
            }
        }

**ResponseFuture**

public void executeInvokeCallback() {
        
            if (invokeCallback != null) {
        
                if (this.executeCallbackOnlyOnce.compareAndSet(false, true)) {
        
                    //执行回调函数
                    invokeCallback.operationComplete(this);
                }
            }
        }

也就是执行这里的回调函数![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L051YW5fRmVuZw_size_16_color_FFFFFF_t_70_pic_center]

## 批量发送消息 ##

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L051YW5fRmVuZw_size_16_color_FFFFFF_t_70_pic_center 1]

public static byte[] encodeMessage(Message message) {
        
            //only need flag, body, properties
            byte[] body = message.getBody();
            int bodyLen = body.length;
            String properties = messageProperties2String(message.getProperties());
            byte[] propertiesBytes = properties.getBytes(CHARSET_UTF8);
            //note properties length must not more than Short.MAX
            short propertiesLength = (short) propertiesBytes.length;
            int sysFlag = message.getFlag();
            int storeSize = 4 // 1 TOTALSIZE
                + 4 // 2 MAGICCOD
                + 4 // 3 BODYCRC
                + 4 // 4 FLAG
                + 4 + bodyLen // 4 BODY
                + 2 + propertiesLength;
            ByteBuffer byteBuffer = ByteBuffer.allocate(storeSize);
            // 1 TOTALSIZE
            byteBuffer.putInt(storeSize);
    
            // 2 MAGICCODE
            byteBuffer.putInt(0);
    
            // 3 BODYCRC
            byteBuffer.putInt(0);
    
            // 4 FLAG
            int flag = message.getFlag();
            byteBuffer.putInt(flag);
    
            // 5 BODY
            byteBuffer.putInt(bodyLen);
            byteBuffer.put(body);
    
            // 6 properties
            byteBuffer.putShort(propertiesLength);
            byteBuffer.put(propertiesBytes);
    
            return byteBuffer.array();
        }

[Link 1]: https://github.com/apache/rocketmq/blob/master/docs/cn/design.md
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L051YW5fRmVuZw_size_16_color_FFFFFF_t_70_pic_center]: /images/20221123/5ac8f3d0293b4ff0b2a40e34b9625703.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L051YW5fRmVuZw_size_16_color_FFFFFF_t_70_pic_center 1]: /images/20221123/55a1bf9f178449a0a589056da9a76f45.png