1. mysql-binlog-connector的简单使用
- 引入依赖：
<dependency>
<groupId>com.github.shyiko</groupId>
<artifactId>mysql-binlog-connector-java</artifactId>
<version>0.16.1</version>
</dependency>
- JAVA的Demo
@Slf4j
public class TestBinlog {
private static final ParserConfig snakeCase;
static {
snakeCase = new ParserConfig();
snakeCase.propertyNamingStrategy = PropertyNamingStrategy.SnakeCase;
}
public static void main(String[] args) throws IOException {
BinaryLogClient client = new BinaryLogClient("localhost", 3306, "root", "123qwe");
EventDeserializer eventDeserializer = new EventDeserializer();
//时间反序列化的格式
// eventDeserializer.setCompatibilityMode(
// EventDeserializer.CompatibilityMode.DATE_AND_TIME_AS_LONG,
// EventDeserializer.CompatibilityMode.CHAR_AND_BINARY_AS_BYTE_ARRAY
// );
client.setEventDeserializer(eventDeserializer);
client.registerEventListener(new BinaryLogClient.EventListener() {
@Override
public void onEvent(Event event) {
EventHeader header = event.getHeader();
EventType eventType = header.getEventType();
System.out.println("监听的事件类型:" + eventType);
if (EventType.isWrite(eventType)) {
//获取事件体
WriteRowsEventData data = event.getData();
log.info(JSON.toJSONString(data));
} else if (EventType.isUpdate(eventType)) {
UpdateRowsEventData data = event.getData();
log.info(JSON.toJSONString(data));
} else if (EventType.isDelete(eventType)) {
DeleteRowsEventData data = event.getData();
log.info(JSON.toJSONString(data));
}
}
});
client.connect();
}
}
注意点：
- client监听维度是数据库级别的监听；
- 可以监听表的DML语句和DDL语句；
2. mysql-binlog-connector的监听事件
源码位置：com.github.shyiko.mysql.binlog.event.EventType
public enum EventType {
/**
* Events of this event type should never occur. Not written to a binary log.
*/
UNKNOWN,
/**
* A descriptor event that is written to the beginning of the each binary log file. (In MySQL 4.0 and 4.1,
* this event is written only to the first binary log file that the server creates after startup.) This event is
* used in MySQL 3.23 through 4.1 and superseded in MySQL 5.0 by {@link #FORMAT_DESCRIPTION}.
*/
START_V3,
/**
* Written when an updating statement is done.
*/
QUERY,
/**
* Written when mysqld stops.
*/
STOP,
/**
* Written when mysqld switches to a new binary log file. This occurs when someone issues a FLUSH LOGS statement or
* the current binary log file becomes larger than max_binlog_size.
*/
ROTATE,
/**
* Written every time a statement uses an AUTO_INCREMENT column or the LAST_INSERT_ID() function; precedes other
* events for the statement. This is written only before a {@link #QUERY} and is not used in case of RBR.
*/
INTVAR,
/**
* Used for LOAD DATA INFILE statements in MySQL 3.23.
*/
LOAD,
/**
* Not used.
*/
SLAVE,
/**
* Used for LOAD DATA INFILE statements in MySQL 4.0 and 4.1.
*/
CREATE_FILE,
/**
* Used for LOAD DATA INFILE statements as of MySQL 4.0.
*/
APPEND_BLOCK,
/**
* Used for LOAD DATA INFILE statements in 4.0 and 4.1.
*/
EXEC_LOAD,
/**
* Used for LOAD DATA INFILE statements as of MySQL 4.0.
*/
DELETE_FILE,
/**
* Used for LOAD DATA INFILE statements in MySQL 4.0 and 4.1.
*/
NEW_LOAD,
/**
* Written every time a statement uses the RAND() function; precedes other events for the statement. Indicates the
* seed values to use for generating a random number with RAND() in the next statement. This is written only
* before a {@link #QUERY} and is not used in case of RBR.
*/
RAND,
/**
* Written every time a statement uses a user variable; precedes other events for the statement. Indicates the
* value to use for the user variable in the next statement. This is written only before a {@link #QUERY} and
* is not used in case of RBR.
*/
USER_VAR,
/**
* A descriptor event that is written to the beginning of the each binary log file.
* This event is used as of MySQL 5.0; it supersedes {@link #START_V3}.
*/
FORMAT_DESCRIPTION,
/**
* Generated for a commit of a transaction that modifies one or more tables of an XA-capable storage engine.
* Normal transactions are implemented by sending a {@link #QUERY} containing a BEGIN statement and a {@link #QUERY}
* containing a COMMIT statement (or a ROLLBACK statement if the transaction is rolled back).
*/
XID,
/**
* Used for LOAD DATA INFILE statements as of MySQL 5.0.
*/
BEGIN_LOAD_QUERY,
/**
* Used for LOAD DATA INFILE statements as of MySQL 5.0.
*/
EXECUTE_LOAD_QUERY,
/**
* This event precedes each row operation event. It maps a table definition to a number, where the table definition
* consists of database and table names and column definitions. The purpose of this event is to enable replication
* when a table has different definitions on the master and slave. Row operation events that belong to the same
* transaction may be grouped into sequences, in which case each such sequence of events begins with a sequence
* of TABLE_MAP events: one per table used by events in the sequence.
* Used in case of RBR.
*/
TABLE_MAP,
/**
* Describes inserted rows (within a single table).
* Used in case of RBR (5.1.0 - 5.1.15).
*/
PRE_GA_WRITE_ROWS,
/**
* Describes updated rows (within a single table).
* Used in case of RBR (5.1.0 - 5.1.15).
*/
PRE_GA_UPDATE_ROWS,
/**
* Describes deleted rows (within a single table).
* Used in case of RBR (5.1.0 - 5.1.15).
*/
PRE_GA_DELETE_ROWS,
/**
* Describes inserted rows (within a single table).
* Used in case of RBR (5.1.16 - mysql-trunk).
*/
WRITE_ROWS,
/**
* Describes updated rows (within a single table).
* Used in case of RBR (5.1.16 - mysql-trunk).
*/
UPDATE_ROWS,
/**
* Describes deleted rows (within a single table).
* Used in case of RBR (5.1.16 - mysql-trunk).
*/
DELETE_ROWS,
/**
* Used to log an out of the ordinary event that occurred on the master. It notifies the slave that something
* happened on the master that might cause data to be in an inconsistent state.
*/
INCIDENT,
/**
* Sent by a master to a slave to let the slave know that the master is still alive. Not written to a binary log.
*/
HEARTBEAT,
/**
* In some situations, it is necessary to send over ignorable data to the slave: data that a slave can handle in
* case there is code for handling it, but which can be ignored if it is not recognized.
*/
IGNORABLE,
/**
* Introduced to record the original query for rows events in RBR.
*/
ROWS_QUERY,
/**
* Describes inserted rows (within a single table).
* Used in case of RBR (5.1.18+).
*/
EXT_WRITE_ROWS,
/**
* Describes updated rows (within a single table).
* Used in case of RBR (5.1.18+).
*/
EXT_UPDATE_ROWS,
/**
* Describes deleted rows (within a single table).
* Used in case of RBR (5.1.18+).
*/
EXT_DELETE_ROWS,
/**
* Global Transaction Identifier.
*/
GTID,
ANONYMOUS_GTID,
PREVIOUS_GTIDS,
TRANSACTION_CONTEXT,
VIEW_CHANGE,
/**
* Prepared XA transaction terminal event similar to XID except that it is specific to XA transaction.
*/
XA_PREPARE;
public static boolean isRowMutation(EventType eventType) {
return EventType.isWrite(eventType) ||
EventType.isUpdate(eventType) ||
EventType.isDelete(eventType);
}
public static boolean isWrite(EventType eventType) {
return eventType == PRE_GA_WRITE_ROWS ||
eventType == WRITE_ROWS ||
eventType == EXT_WRITE_ROWS;
}
public static boolean isUpdate(EventType eventType) {
return eventType == PRE_GA_UPDATE_ROWS ||
eventType == UPDATE_ROWS ||
eventType == EXT_UPDATE_ROWS;
}
public static boolean isDelete(EventType eventType) {
return eventType == PRE_GA_DELETE_ROWS ||
eventType == DELETE_ROWS ||
eventType == EXT_DELETE_ROWS;
}
}
2.1 DML语句的事件
DML操作是指对数据库中表记录的操作，主要包括表记录的插入（insert）、更新（update）、删除（delete）和查询（select），是开发人员日常使用最频繁的操作。
监听的事件类型:ANONYMOUS_GTID
监听的事件类型:QUERY
监听的事件类型:TABLE_MAP
监听的事件类型:EXT_UPDATE_ROWS
监听的事件类型:XID
2.2 DDL语句事件
数据库模式定义语言DDL(Data Definition Language)，是用于描述数据库中要存储的现实世界实体的语言。主要由create（添加）、alter（修改）、drop（删除）和 truncate（删除） 四个关键字完成。
例如：alter table test_table1 add column school_name varchar(255) not null;
监听的事件类型:ANONYMOUS_GTID
监听的事件类型:QUERY
2.3 端点续传事件
当binlog服务上线之后，服务就会在log文件的最新position处进行监听。格式如下：
Connected to localhost:3306 at mysql-bin.000013/1075 (sid:65535, cid:7)
但由于binlog服务下线，重新启动后，默认又开始在最新position处进行监听。会丢失一部门binlog的事件。故，每次事件均需要记录当前的position位置。重新建立client端时，使用记录的position位置。
故需要监听ROTATE事件
//当mysqld切换到新的二进制日志文件时写入。1. 当有人发出一个FLUSH LOGS 语句。或者当前二进制日志文件变大，超过max_binlog_size。
ROTATE,
2.4 不计入position更新(断点续传)的事件
-
FORMAT_DESCRIPTION类型为binlog起始时间。 -
HEARTBEAT为心跳检测事件，不会写入master的binlog，记录该事件的position会导致重启时报错。
3. 高可用&&端点续传
3.1 对于高可用
mysql-binlog-connector监听只能是单点服务。但是需要保证高可用，即某台机器挂掉，但是不能影响服务。
实现思路：部署多台机器，只有一台机器能监听binlog，当这台机器挂掉后，其他机器抢夺binlog的监听。
可以借助Zookeeper，也可以借助Redis实现。
3.2 对于端点续传。
mysql-binlog-connector没有帮我们实现断点续传，所以需要我们自己记录每个事件的position的位置。当重启服务时，自动读取记录的position位置进行识别。
import lombok.Data;
@Data
public class SyncConfig {
/**
* mysql的host配置
*/
String host;
/**
* mysql的port配置
*/
Integer port;
/**
* mysql的userName配置
*/
String userName;
/**
* mysql的password配置
*/
String password;
}
/**
* 处理binlog位点信息接口，实现该接口创建自定义位点处理类
*/
public interface IPositionHandler {
BinlogPositionEntity getPosition(SyncConfig syncConfig);
void savePosition(SyncConfig syncConfig, BinlogPositionEntity binlogPositionEntity);
}
/**
* 默认的端点续传存储器（需要借助分布式存储——此处可以保存到Redis中）
*/
public class DefaultPositionHandler implements IPositionHandler {
private Map<String, String> cache = new ConcurrentHashMap<>();
@Override
public BinlogPositionEntity getPosition(SyncConfig syncConfig) {
return JSON.parseObject(cache.get(generateKey(syncConfig)), BinlogPositionEntity.class);
}
@Override
public void savePosition(SyncConfig syncConfig, BinlogPositionEntity binlogPositionEntity) {
cache.put(generateKey(syncConfig), JSON.toJSONString(binlogPositionEntity));
}
/**
* @param syncConfig 参数配置
* @return 生成的key
*/
private String generateKey(SyncConfig syncConfig) {
return syncConfig.getHost() + ":" + syncConfig.getPort();
}
}
/**
* 记录当前监听位置的事件
*
*/
@Data
public class BinlogPositionEntity {
/**
* binlog文件的名字
*/
private String binlogName;
/**
* binlog文件的位置
*/
private Long position;
/**
* binlog的服务id
*/
private Long serverId;
}
案例demo：
@Slf4j
public class TestBinlog {
private static final SyncConfig syncConfig;
private static IPositionHandler positionHandler = new DefaultPositionHandler();
/**
* 项目启动读取的配置
*/
static {
syncConfig = new SyncConfig();
syncConfig.setHost("localhost");
syncConfig.setPort(3306);
syncConfig.setUserName("root");
syncConfig.setPassword("123qwe");
}
public static void main(String[] args) throws IOException {
BinaryLogClient client = new BinaryLogClient(syncConfig.getHost(), syncConfig.getPort(), syncConfig.getUserName(), syncConfig.getPassword());
EventDeserializer eventDeserializer = new EventDeserializer();
//时间反序列化的格式
// eventDeserializer.setCompatibilityMode(
// EventDeserializer.CompatibilityMode.DATE_AND_TIME_AS_LONG,
// EventDeserializer.CompatibilityMode.CHAR_AND_BINARY_AS_BYTE_ARRAY
// );
client.setEventDeserializer(eventDeserializer);
//设置serverId，不同的集群，机器的serverId不能相同。
client.setServerId(getRandomServerId());
//获取position的位置（创建client时，读取当前记录的postion）
BinlogPositionEntity binlogPositionEntity = positionHandler.getPosition(syncConfig);
if (binlogPositionEntity != null &&
binlogPositionEntity.getBinlogName() != null &&
binlogPositionEntity.getPosition() != null) {
client.setBinlogFilename(binlogPositionEntity.getBinlogName());
client.setBinlogPosition(binlogPositionEntity.getPosition());
}
client.registerEventListener(new BinaryLogClient.EventListener() {
@Override
public void onEvent(Event event) {
EventHeader header = event.getHeader();
EventType eventType = header.getEventType();
System.out.println("监听的事件类型:" + eventType);
/*
* 不计入position更新的事件类型
* FORMAT_DESCRIPTION类型为binlog起始时间
* HEARTBEAT为心跳检测事件，不会写入master的binlog，记录该事件的position会导致重启时报错
*/
List<EventType> excludePositionEventType = new ArrayList<>();
excludePositionEventType.add(EventType.FORMAT_DESCRIPTION);
excludePositionEventType.add(EventType.HEARTBEAT);
if (!excludePositionEventType.contains(eventType)) {
BinlogPositionEntity binlogPositionEntity = new BinlogPositionEntity();
//处理rotate事件,这里会替换调binlog fileName
if (event.getHeader().getEventType().equals(EventType.ROTATE)) {
RotateEventData rotateEventData = (RotateEventData) event.getData();
binlogPositionEntity.setBinlogName(rotateEventData.getBinlogFilename());
binlogPositionEntity.setPosition(rotateEventData.getBinlogPosition());
binlogPositionEntity.setServerId(event.getHeader().getServerId());
} else { //统一处理事件对应的binlog position
//在Redis中获取获取binlog的position配置
binlogPositionEntity = positionHandler.getPosition(syncConfig);
EventHeaderV4 eventHeaderV4 = (EventHeaderV4) event.getHeader();
binlogPositionEntity.setPosition(eventHeaderV4.getPosition());
binlogPositionEntity.setServerId(event.getHeader().getServerId());
}
//将最新的配置保存到Redis中
log.info("保存的数据{}", JSON.toJSONString(binlogPositionEntity));
positionHandler.savePosition(syncConfig, binlogPositionEntity);
//todo 解析结果
}
}
});
client.connect();
}
private static long getRandomServerId() {
try {
return SecureRandom.getInstanceStrong().nextLong();
} catch (NoSuchAlgorithmException e) {
return RandomUtils.nextLong();
}
}
}
推荐的的binlog监听工具：
https://github.com/dothetrick/binlogportal
