20221010_rocketmq通信协议体学习笔记.md
1概述
1.1总体数据结构RemotingCommand(所属包remoting )
image-20221017071358222.png
消息传输过程中的对数据内容的封装类，结构如下分四部分
1、消息长度：消息的总长度，int类型，四个字节存储,注意:nettyEncoder编码时,totalLength为后面三部分组成:length(header)+length(headDatas)+length(bodyDatas)；
2、序列化类型&&头部长度：int类型，第一个字节表示序列化类型，后面三个字节表示消息头长度；markProtocolType
3、消息头数据：经过序列化后的消息头数据；
4、消息主体数据：经过编码后消息主体的二进制字节数据内容
1.1.1总体结构业务请求响应码code
1.1.1.1RequestCode(所属包common)
1.1.1.2ResponseCode(所属包common)
ResponseCode继承 RemotingSysResponseCode(所属包remoting)
1.1.2flag
flag意义重大,flag最后一位代码请求与响应类型。倒数第二位代码是否是单向请求。
private static final int RPC_TYPE = 0; // 0, REQUEST_COMMAND,代表2^0
private static final int RPC_ONEWAY = 1; // 0, RPC,代表2^1
请求响应标识 RemotingCommandType (所属包remoting)
public enum RemotingCommandType {
REQUEST_COMMAND,
RESPONSE_COMMAND;
}
1.1.2.1协议类型及请求头
// source:协议头长度 正常高位不会有这么大的数值
// 0x 01 00 00 00-->16^5=1677 7216
public static byte[] markProtocolType(int source, SerializeType type) {
byte[] result = new byte[4];
result[0] = type.getCode(); // 第一个字节存储序列化类型
result[1] = (byte) ((source >> 16) & 0xFF); // 低位
result[2] = (byte) ((source >> 8) & 0xFF);
result[3] = (byte) (source & 0xFF);
return result;
}
1.1.2.2单向请求设置及判断
// 单向时设置flag倒数第二位为:1
public void markOnewayRPC() {
int bits = 1 << RPC_ONEWAY;
this.flag |= bits;
}
// 判断是否是oneWay
@JSONField(serialize = false)
public boolean isOnewayRPC() {
int bits = 1 << RPC_ONEWAY;
return (this.flag & bits) == bits;
}
1.1.2.3设置响应类型
// 非oneWay时,解析后设置最后一位不是1变为1,0,请求;1,响应
public void markResponseType() {
int bits = 1 << RPC_TYPE;
this.flag |= bits;
}
1.1.3请求流水号opaque
private int opaque = requestId.getAndIncrement();
1.2数据序列化(针对RemotingCommand.Header)
1.2.1子数据序列化类型SerializeType(所属包remoting)
public enum SerializeType {
JSON((byte) 0),
ROCKETMQ((byte) 1);
private byte code;
SerializeType(byte code) {
this.code = code;
}
public static SerializeType valueOf(byte code) {
for (SerializeType serializeType : SerializeType.values()) {
if (serializeType.getCode() == code) {
return serializeType;
}
}
return null;
}
public byte getCode() {
return code;
}
}
1.2.2请求头序列化
支持2种序列化方式JSON(默认)、RocketMQSerializable
1.2.2.1headerEncode
private byte[] headerEncode() {
this.makeCustomHeaderToNet();
if (SerializeType.ROCKETMQ == serializeTypeCurrentRPC) {
return RocketMQSerializable.rocketMQProtocolEncode(this);
} else {
return RemotingSerializable.encode(this);
}
}
1.2.2.2headerDecode
private static RemotingCommand headerDecode(byte[] headerData, SerializeType type) {
switch (type) {
case JSON:
RemotingCommand resultJson = RemotingSerializable.decode(headerData, RemotingCommand.class);
resultJson.setSerializeTypeCurrentRPC(type);
return resultJson;
case ROCKETMQ:
RemotingCommand resultRMQ = RocketMQSerializable.rocketMQProtocolDecode(headerData);
resultRMQ.setSerializeTypeCurrentRPC(type);
return resultRMQ;
default:
break;
}
return null;
}
1.2.3业务数据序列化(抽象类RemotingSerializable所属包remoting)
common extends remoting
org.apache.rocketmq.common.protocol.bodyRegisterBrokerBody
抽象类,内置1种序列化方式JSON。作为协议体body,继承 RemotingSerializable。
public class Message implements Serializable {
public class HeartbeatData extends RemotingSerializable {
// 不需要实现序JDK列化吗
public class RegisterBrokerBody extends RemotingSerializable {
public byte[] encode(boolean compress) {
if (!compress) {
return super.encode();
}
image-20221010081407120.png
// E:\workdirectory\OpenSourceStudy\rocketmq-all-4.8.0-source-release\acl\src\test\java\org\apache\rocketmq\acl\plain\PlainAccessValidatorTest.java
@Test
public void validateHeartBeatTest() {
HeartbeatData heartbeatData=new HeartbeatData();
Set<ProducerData> producerDataSet=new HashSet<>();
Set<ConsumerData> consumerDataSet=new HashSet<>();
Set<SubscriptionData> subscriptionDataSet=new HashSet<>();
ProducerData producerData=new ProducerData();
producerData.setGroupName("producerGroupA");
ConsumerData consumerData=new ConsumerData();
consumerData.setGroupName("consumerGroupA");
SubscriptionData subscriptionData=new SubscriptionData();
subscriptionData.setTopic("topicC");
producerDataSet.add(producerData);
consumerDataSet.add(consumerData);
subscriptionDataSet.add(subscriptionData);
consumerData.setSubscriptionDataSet(subscriptionDataSet);
heartbeatData.setProducerDataSet(producerDataSet);
heartbeatData.setConsumerDataSet(consumerDataSet);
// 1.构建RemotingCommand(不带请求头)
RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.HEART_BEAT,null);
// 2.这里对业务数据 HeartbeatData进行了编码,并放到协议体里
remotingCommand.setBody(heartbeatData.encode());
aclClient.doBeforeRequest("", remotingCommand);
// 3.对RemotingCommand进行编码(head编码、body编码、最后组合)
ByteBuffer buf = remotingCommand.encode();
buf.getInt();
buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
buf.position(0);
PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "192.168.0.1:9876");
plainAccessValidator.validate(accessResource);
}
// E:\workdirectory\OpenSourceStudy\rocketmq-all-4.8.0-source-release\remoting\src\main\java\org\apache\rocketmq\remoting\protocol\RemotingCommand.java
public ByteBuffer encode() {
// 1> header length size
int length = 4;
// 2> header data length
byte[] headerData = this.headerEncode();
length += headerData.length;
// 3> body data length
if (this.body != null) {
length += body.length;
}
ByteBuffer result = ByteBuffer.allocate(4 + length);
// length total,包括自己
result.putInt(length);
// header length
result.put(markProtocolType(headerData.length, serializeTypeCurrentRPC));
// header data
result.put(headerData);
// body data;
if (this.body != null) {
result.put(this.body);
}
result.flip();
return result;
}
1.3底层包的依赖关系
image-20221013062034884.png
1.3.1服务端
image-20221013063301484.png
1.3.2客户端
image-20221013063337696.png
1.4日志的类关系
image-20221013062230444.png
image-20221013062301688.png
1.5粘包的处理方式
1.5.1编码NettyEncoder(所属包remoting)
@ChannelHandler.Sharable:发完即销毁, 所以可以共享,
@ChannelHandler.Sharable
public class NettyEncoder extends MessageToByteEncoder<RemotingCommand> {
@Override
public void encode(ChannelHandlerContext ctx, RemotingCommand remotingCommand, ByteBuf out)
throws Exception {
try {
// 1.encodeHeader&setHeader
ByteBuffer header = remotingCommand.encodeHeader();
out.writeBytes(header);
// 2.set body
byte[] body = remotingCommand.getBody();
if (body != null) {
out.writeBytes(body);
}
} catch (Exception e) {
log.error("encode exception, " + RemotingHelper.parseChannelRemoteAddr(ctx.channel()), e);
if (remotingCommand != null) {
log.error(remotingCommand.toString());
}
RemotingUtil.closeChannel(ctx.channel());
}
1.5.1.1RemotingCommand.encodeHeader vs RemotingCommand.encode
主要区别在于第一个 4字节里面的长度域,感觉ali也挺坑的,不注意就被绕。
encodeHeader :多了4个字节(第一个字节序列化类型+后面3个字节消息头长度)
encode:少了4个字节(第一个字节序列化类型+后面3个字节消息头长度)
1.5.2解码NettyDecoder(所属包remoting)
public class NettyDecoder extends LengthFieldBasedFrameDecoder {
public NettyDecoder() {
// lengthFieldLength:4
// lengthAdjustment:0,
// 当Netty利用lengthFieldOffset（偏移位）和lengthFieldLength（Length字段长度）成功读出Length字段的值后，
// Netty默认 认为这个值是指从Length字段之后，到包结束一共还有多少字节，
// 如果这个值是13，那么Netty就会再等待13个Byte的数据到达后
// initialBytesToStrip:4,相对整个消息,需要跳过前面的字节数
super(FRAME_MAX_LENGTH, 0, 4, 0, 4);
}
@Override
public Object decode(ChannelHandlerContext ctx, ByteBuf in) throws Exception {
ByteBuf frame = null;
try {
// 1.得到原始帧(预解)
frame = (ByteBuf) super.decode(ctx, in);
if (null == frame) {
return null;
}
// 2.获取只读的nioBuffer
ByteBuffer byteBuffer = frame.nioBuffer();
// 3.解码得到 RemotingCommand对象
return RemotingCommand.decode(byteBuffer);
} catch (Exception e) {
log.error("decode exception, " + RemotingHelper.parseChannelRemoteAddr(ctx.channel()), e);
RemotingUtil.closeChannel(ctx.channel());
} finally {
// 4.释放ByteBuf
if (null != frame) {
frame.release();
}
}
return null;
}
}
1.6消息通信三种方式CommunicationMode(所属包client)
client包定义了具体的生成与消费实现。
public enum CommunicationMode {
SYNC,
ASYNC,
ONEWAY,
}
1.6.1同步（sync）本质基于countDownLatch
只支持串行操作,设定待超时时间
同步原理:
客户端构建ResponseFuture对象(内置countDownLatch)，放入responseTable缓存中。
-
利用channel.writeAndFlush(request).addListener机制开启监听,操作完成则修改responseFuture对象属性sendRequestOK、
删除responseTable里的请求流水号、修改responseFuture对象属性cause,和唤醒同步锁。
responseFuture.waitResponse(timeoutMillis)进行同步锁超时等待。
/**
* This map caches all on-going requests.
*/
protected final ConcurrentMap<Integer /* opaque */, ResponseFuture> responseTable =
new ConcurrentHashMap<Integer, ResponseFuture>(256);
// C005mynettysocket3demo\mynettyremoting\src\main\java\com\kikop\remoting\netty\NettyRemotingAbstract.java
public RemotingCommand invokeSyncImpl(final Channel channel, final RemotingCommand request,
final long timeoutMillis)
throws InterruptedException, RemotingSendRequestException, RemotingTimeoutException {
final ResponseFuture responseFuture = new ResponseFuture(channel, opaque, timeoutMillis, null, null);
this.responseTable.put(opaque, responseFuture);
final SocketAddress addr = channel.remoteAddress();
channel.writeAndFlush(request).addListener(new ChannelFutureListener() {
@Override
public void operationComplete(ChannelFuture f) throws Exception {
if (f.isSuccess()) {
responseFuture.setSendRequestOK(true);
return;
} else {
responseFuture.setSendRequestOK(false);
}
responseTable.remove(opaque);
responseFuture.setCause(f.cause());
responseFuture.putResponse(null);
log.warn("send a request command to channel <" + addr + "> failed.");
}
});
public class Message implements Serializable {
// E:\workdirectory\OpenSourceStudy\rocketmq-all-4.8.0-source-release\client\src\main\java\org\apache\rocketmq\client\impl\producer\DefaultMQProducerImpl.java
public SendResult send(Message msg, MessageQueue mq, long timeout)
throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
long beginStartTime = System.currentTimeMillis();
this.makeSureStateOK();
Validators.checkMessage(msg, this.defaultMQProducer);
if (!msg.getTopic().equals(mq.getTopic())) {
throw new MQClientException("message's topic not equal mq's topic", null);
}
long costTime = System.currentTimeMillis() - beginStartTime;
if (timeout < costTime) {
throw new RemotingTooMuchRequestException("call timeout");
}
// 指定发送方式
return this.sendKernelImpl(msg, mq, CommunicationMode.SYNC, null, null, timeout);
}
1.6.2异步（async）本质基于countDownLatch
可以并行任务处理，客户端CountDownLatch等AQS同步锁机制。
1.6.3单向（oneway）
客户端不需等到返回值。
1.6.3.1判断是否是单向
private static final int RPC_ONEWAY = 1; // 0, RPC
@JSONField(serialize = false)
public boolean isOnewayRPC() {
int bits = 1 << RPC_ONEWAY;
return (this.flag & bits) == bits;
}
1.7NettyRemotingServer
image-20221015205327138.png
@Override
public void registerProcessor(int requestCode, NettyRequestProcessor processor, ExecutorService executor) {
ExecutorService executorThis = executor;
if (null == executor) {
executorThis = this.publicExecutor;
}
Pair<NettyRequestProcessor, ExecutorService> pair = new Pair<NettyRequestProcessor, ExecutorService>(processor, executorThis);
this.processorTable.put(requestCode, pair);
}
1.7.1.服务端处理器的注册
// E:\workdirectory\OpenSourceStudy\rocketmq-all-4.8.0-source-release\broker\src\main\java\org\apache\rocketmq\broker\BrokerStartup.java
public static void main(String[] args) {
start(createBrokerController(args));
public static BrokerController createBrokerController(String[] args) {
// E:\workdirectory\OpenSourceStudy\rocketmq-all-4.8.0-source-release\broker\src\main\java\org\apache\rocketmq\broker\BrokerController.java
public boolean initialize() throws CloneNotSupportedException {
public void registerProcessor() {
this.remotingServer.registerProcessor(RequestCode.SEND_MESSAGE, sendProcessor, this.sendMessageExecutor);
1.7.2.客户端处理器的注册
// E:\workdirectory\OpenSourceStudy\rocketmq-all-4.8.0-source-release\client\src\main\java\org\apache\rocketmq\client\impl\consumer\DefaultMQPushConsumerImpl.java
public synchronized void start() throws MQClientException {
this.mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(this.defaultMQPushConsumer, this.rpcHook);
// E:\workdirectory\OpenSourceStudy\rocketmq-all-4.8.0-source-release\client\src\main\java\org\apache\rocketmq\client\impl\MQClientManager.java
public MQClientInstance getOrCreateMQClientInstance(final ClientConfig clientConfig, RPCHook rpcHook) {
String clientId = clientConfig.buildMQClientId();
MQClientInstance instance = this.factoryTable.get(clientId);
if (null == instance) {
instance =
new MQClientInstance(clientConfig.cloneClientConfig(),
this.factoryIndexGenerator.getAndIncrement(), clientId, rpcHook);
// E:\workdirectory\OpenSourceStudy\rocketmq-all-4.8.0-source-release\client\src\main\java\org\apache\rocketmq\client\impl\factory\MQClientInstance.java
public MQClientInstance(ClientConfig clientConfig, int instanceIndex, String clientId, RPCHook rpcHook) {
this.mQClientAPIImpl = new MQClientAPIImpl(this.nettyClientConfig, this.clientRemotingProcessor, rpcHook, clientConfig);
// E:\workdirectory\OpenSourceStudy\rocketmq-all-4.8.0-source-release\client\src\main\java\org\apache\rocketmq\client\impl\MQClientAPIImpl.java
public MQClientAPIImpl(final NettyClientConfig nettyClientConfig,
final ClientRemotingProcessor clientRemotingProcessor,
RPCHook rpcHook, final ClientConfig clientConfig) {
this.remotingClient.registerProcessor(RequestCode.CHECK_TRANSACTION_STATE, this.clientRemotingProcessor, null);
1.7.3服务端接收到请求的响应处理流程
1.首先channelRead0,processMessageReceived
case REQUEST_COMMAND:
processRequestCommand(ctx, cmd);
2.根据请求码获取匹配的业务处理器
3.创建一个runnable,具体逻辑构建RemotingResponseCallback,入参为RemotingCommand(flag此时还是原来的)
如果不是单向,则设置响应类型,并写入字节流到响应通道
1.8优秀的架构设计思想总结
1.8.1服务端
- 流控处理,服务端通过Semaphore来设置单向通信和异步通信执行的任务数量,否则抛出RemotingTimeoutException。
- 服务端公共请求处理器,服务端NettyRemotingServer创建一个公共的线程池执行器(jdk ExecutorService publicExecutor,构造函数触发),用于处理未匹配的业务处理器和registerDefaultProcessor(初始化化时)。
- 服务端业务请求处理器,服务端通过注册处理器AsyncNettyRequestProcessor,当processRequestCommand时,针对某个业务请求码进行线程池细粒度调度。
1.8.2客户端
- 超时处理,客户端同步发送请求带指定的超时时间,不至于一直傻等。
- 客户端业务请求处理器,客户通过注册处理器AsyncNettyRequestProcessor,当processRequest时,针对某个业务请求码进行线程池的粒度化合理调度。
2代码示例
2.1协议接口编解码单元测试
package com.kikop;
import com.kikop.remoting.exception.RemotingConnectException;
import com.kikop.remoting.exception.RemotingSendRequestException;
import com.kikop.remoting.exception.RemotingTimeoutException;
import com.kikop.remoting.netty.NettyDecoder;
import com.kikop.remoting.netty.NettyEncoder;
import com.kikop.remoting.protocol.RemotingCommand;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.embedded.EmbeddedChannel;
import org.junit.Test;
public class NettyEncoderTest {
/**
* 模仿 Netty 进行编解码接口测试
*
* @throws InterruptedException
* @throws RemotingConnectException
* @throws RemotingSendRequestException
* @throws RemotingTimeoutException
*/
@Test
public void testObjEncoderAndDecoder() throws InterruptedException, RemotingConnectException,
RemotingSendRequestException, RemotingTimeoutException {
// 1.构造 RemotingCommand
RequestHeader requestHeader = new RequestHeader();
requestHeader.setCount(1);
requestHeader.setMessageTitle("Welcome");
RemotingCommand request = RemotingCommand.createRequestCommand(10086, requestHeader);
// 2.初始化处理器
ChannelInitializer channelInitializer = new ChannelInitializer<EmbeddedChannel>() {
protected void initChannel(EmbeddedChannel ch) {
ch.pipeline().addLast(new NettyEncoder());
ch.pipeline().addLast(new NettyDecoder());
}
};
// 3.构建embeddedChannel
EmbeddedChannel embeddedChannel = new EmbeddedChannel(channelInitializer);
// 4.编码
// 将请求对象转为请求字节流
embeddedChannel.writeOutbound(request); // trigger call encode
// 本质:UnpooledUnsafeNoCleanerDirectByteBuf
// class io.netty.buffer.UnpooledUnsafeNoCleanerDirectByteBuf
Object byteBuf = embeddedChannel.readOutbound(); // 得到字节流
// 5.解码
// 将请求字节流转为请求对象(完美)
// class com.kikop.remoting.protocol.RemotingCommand
embeddedChannel.writeInbound(byteBuf); // trigger call decode
Object objResult = embeddedChannel.readInbound(); // 得到请求对象
}
}
参考
1聊聊rocketmq的NettyEncoder及NettyDecoder
https://blog.csdn.net/weixin_33850015/article/details/88764614
2Netty在RocketMQ中的应用----编解码
3RocketMQ源码阅读（二）RemotingCommand、NettyEncoder和NettyDecoder
4大端小端字节序，网络字节序，Intel字节序
https://blog.csdn.net/yangzhengzheng95/article/details/122636264
