SpringBoot2 整合Apache Zookeeper集群

两个半月前,整理了一篇 SpringBoot集成Zookeeper,实现服务的注册与发现踩坑,过去个把月,再次翻看时,才发现,代码结构很不清晰,惨不忍睹。这次二次整理,代码上简洁了很多,功能上也更加完善了。

本demo运行的前提,本地已经配置了zookeeper集群,最少三个节点。没有安装的,可以参考前面的文章安装一下。本demo依旧是在SpringBoot版本上完成的,主要实现的功能有:

  1. 服务的注册
  2. 服务节点的监控
  3. 服务的负载均衡
  4. 服务最佳节点的获取

首先,在pom文件中引入架包;

<dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>
<!-- zookeeper引用架包start --> 
        <dependency>
              <groupId>org.apache.zookeeper</groupId>
              <artifactId>zookeeper</artifactId>
              <version>3.4.8</version>  
              <exclusions>
                  <exclusion>
                      <groupId>log4j</groupId>
                      <artifactId>log4j</artifactId>
                  </exclusion>
                  <exclusion>
                      <groupId>org.slf4j</groupId>
                      <artifactId>slf4j-api</artifactId>
                  </exclusion>
                  <exclusion>
                      <groupId>org.slf4j</groupId>
                      <artifactId>slf4j-log4j12</artifactId>
                  </exclusion>
              </exclusions>         
          </dependency>
          <dependency>
              <groupId>org.apache.curator</groupId>
              <artifactId>curator-framework</artifactId>
              <version>4.0.0</version>            
          </dependency>
          <dependency>
              <groupId>org.apache.curator</groupId>
              <artifactId>curator-recipes</artifactId>
              <version>4.2.0</version>
          </dependency>
        <!-- zookeeper引用架包end -->
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.62</version>
        </dependency>

第二,配置文件:

server.port: 8400
#zookeeper
zookeeper.register.address=127.0.0.1:2181,127.0.0.1:2182,127.0.0.1:2183
zookeeper.base.sleep.time.ms=1000
zookeeper.max.retries=3
zookeeper.register.node=/test/nodes
zookeeper.register.pathPrefix=/test/nodes/seq-
curator-default-session-timeout=64000

minSessionTimeout=64000
maxSessionTimeout=120000

第三,Config配置文件;

import javax.annotation.PostConstruct;

import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.retry.ExponentialBackoffRetry;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

/**
 * zookeeper连接配置
 * @author 程就人生
 * @date 2020年1月13日
 */
@Configuration
public class ZookeeperConfig {
    
    //zookeeper连接地址
    @Value("${zookeeper.register.address}")
    private String strZkAddress;
    
    //连接初始时间
    @Value("${zookeeper.base.sleep.time.ms}")
    private int strBaseSleepTimeMs;
    
    //重试次数
    @Value("${zookeeper.max.retries}")
    private int strMaxRetries;
    
    @Value("${zookeeper.register.node}")
    private String strManagerPath;
    
    @Value("${zookeeper.register.pathPrefix}")
    private String strPathPrefix;
    
    public static String managerPath;
    
    public static String pathPrefix;
    
    public static String zkAddress;
    
    private static int baseSleepTimeMs;
    
    private static int maxRetries;
    
    /**
     * 静态变量初始化,postContruct的作用
     * 需要执行的方法,在完成依赖项注入后,执行任何初始化
     * 这里用于从配置文件里获取配置,同时保证CuratorFramework只有一个实例
     */
    @PostConstruct
    private void init(){
        zkAddress = strZkAddress;
        baseSleepTimeMs = strBaseSleepTimeMs;
        maxRetries = strMaxRetries;
        managerPath = strManagerPath;
        pathPrefix = strPathPrefix;
    }
    
    /**
     * 创建CuratorFramework实例,全局唯一
     * @return CuratorFramework 实例
     */
    public static CuratorFramework createInstance(){
        
        // 重试策略:第一次重试等待1s,第二次重试等待2s,第三次重试等待4s
        // 第一个参数:等待时间的基础单位,单位为毫秒
        // 第二个参数:最大重试次数
        ExponentialBackoffRetry retryPolicy = new ExponentialBackoffRetry(baseSleepTimeMs, maxRetries);             
        // 第一个参数:zk的连接地址
        // 第二个参数:重试策略
        CuratorFramework curatorFramework = CuratorFrameworkFactory.newClient(zkAddress, retryPolicy);  
        
        return curatorFramework;
    }
    
    /**
     * 非全局唯一,获取节点用,于上面的不同
     * initMethod字段表示,start()方法在createInstance()方法执行完毕后执行
     * @return CuratorFramework 实例
     */
    @Bean(value="zkClient",initMethod="start")
    public CuratorFramework createInstance1(){
        // 重试策略:第一次重试等待1s,第二次重试等待2s,第三次重试等待4s
        // 第一个参数:等待时间的基础单位,单位为毫秒
        // 第二个参数:最大重试次数
        ExponentialBackoffRetry retryPolicy = new ExponentialBackoffRetry(baseSleepTimeMs, maxRetries);             
        // 第一个参数:zk的连接地址
        // 第二个参数:重试策略
        CuratorFramework curatorFramework = CuratorFrameworkFactory.newClient(zkAddress, retryPolicy);  
        
        return curatorFramework;
    }
}

第四,核心文件;

import java.io.Serializable;
import java.util.Objects;

/**
 * 节点属性定义
 * @author 程就人生
 * @date 2020年1月13日
 */
public class ImNode implements Comparable<ImNode>, Serializable {

    private static final long serialVersionUID = -499010884211304846L;


    //worker 的Id,zookeeper负责生成
    private long id;

    //服务 的连接数
    private Integer balance = 0;

    //服务 IP
    private String host;

    //服务 端口
    private Integer port;

    public ImNode() {
    }

    public ImNode(String host, Integer port) {
        this.host = host;
        this.port = port;
    }


    @Override
    public String toString() {
        return "ImNode{" +
                "id='" + id + '\'' +
                ",host='" + host + '\'' +
                ", port='" + port + '\'' +
                ",balance=" + balance +
                '}';
    }

    @Override
    public boolean equals(Object o) {
        if (this == o) return true;
        if (o == null || getClass() != o.getClass()) return false;
        ImNode node = (ImNode) o;
//        return id == node.id &&
        return Objects.equals(host, node.host) &&
                Objects.equals(port, node.port);
    }

    @Override
    public int hashCode() {
        return Objects.hash(id, host, port);
    }

    /**
     * 升序排列
     */
    public int compareTo(ImNode o) {
        int weight1 = this.balance;
        int weight2 = o.balance;
        if (weight1 > weight2) {
            return 1;
        } else if (weight1 < weight2) {
            return -1;
        }
        return 0;
    }


    public void incrementBalance() {
        balance++;
    }

    public void decrementBalance() {
        balance--;
    }

    public final long getId() {
        return id;
    }

    public final void setId(long id) {
        this.id = id;
    }

    public final Integer getBalance() {
        return balance;
    }

    public final void setBalance(Integer balance) {
        this.balance = balance;
    }

    public final String getHost() {
        return host;
    }

    public final void setHost(String host) {
        this.host = host;
    }

    public final Integer getPort() {
        return port;
    }

    public final void setPort(Integer port) {
        this.port = port;
    }
}

import org.apache.curator.framework.CuratorFramework;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.data.Stat;
import org.springframework.stereotype.Component;

import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.test.config.ZookeeperConfig;

/**
 * zookeeper节点创建,确保一个服务一个节点(单例模式使用的原因)
 * @author 程就人生
 * @date 2020年1月13日
 */
@Component
public class ImWorker {
    
    private CuratorFramework client;

    private String managerPath;
    
    private String pathPrefix;

    // 保存当前Znode节点的路径,创建后返回
    private String pathRegistered = null;

    private ImNode localNode = null;

    private static ImWorker singleInstance = null;

    /**
     * 取得单例
     * @return
     */
    public static ImWorker getInst() {
        if (null == singleInstance) {
            singleInstance = new ImWorker();
            singleInstance.localNode = new ImNode();
            singleInstance.managerPath = ZookeeperConfig.managerPath;
            singleInstance.pathPrefix = ZookeeperConfig.pathPrefix;
            singleInstance.client = ZookeeperConfig.createInstance();
            singleInstance.client.start();
        }
        return singleInstance;
    }

    /**
     * 私有的午餐构造方法
     */
    private ImWorker() {

    }
    
    /**
     * 在zookeeper中创建临时节点
     */
    public void init() {
        // 如果父节点不存在,创建父节点
        createParentIfNeeded(managerPath);
        // 创建一个 ZNode 节点
        // 节点的 payload 为当前worker 实例
        try {
            byte[] payload = JSONObject.toJSONBytes(localNode, SerializerFeature.WriteEnumUsingToString);

            pathRegistered = client.create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL_SEQUENTIAL)
                    .forPath(pathPrefix, payload);

            // 为node 设置id
            localNode.setId(getId());

        } catch (Exception e) {
            e.printStackTrace();
        }

    }

    /**
     * 设置本地节点
     * @param ip
     * @param port
     */
    public void setLocalNode(String ip, int port) {
        if (localNode == null) {
            localNode = new ImNode();
        }
        localNode.setHost(ip);
        localNode.setPort(port);
    }

    /**
     * 取得IM 节点编号
     * @return 编号
     */
    public long getId() {

        return getIdByPath(pathRegistered);

    }

    /**
     * 取得IM 节点编号
     * @return 编号
     * @param path 路径
     */
    public long getIdByPath(String path) {
        String sid = null;
        if (null == path) {
            throw new RuntimeException("节点路径有误");
        }
        int index = path.lastIndexOf(pathPrefix);
        if (index >= 0) {
            index += pathPrefix.length();
            sid = index <= path.length() ? path.substring(index) : null;
        }

        if (null == sid) {
            throw new RuntimeException("节点ID获取失败");
        }

        return Long.parseLong(sid);

    }

    /**
     * 增加负载,表示有用户登录成功
     * @return 成功状态
     */
    public boolean incBalance() {
        if (null == localNode) {
            throw new RuntimeException("还没有设置Node 节点");
        }
        // 增加负载:增加负载,并写回zookeeper
        while (true) {
            try {
                localNode.incrementBalance();
                byte[] payload = JSONObject.toJSONBytes(localNode, SerializerFeature.WriteEnumUsingToString);
                client.setData().forPath(pathRegistered, payload);
                return true;
            } catch (Exception e) {
                return false;
            }
        }

    }

    /**
     * 减少负载,表示有用户下线,写回zookeeper
     * @return 成功状态
     */
    public boolean decrBalance() {
        if (null == localNode) {
            throw new RuntimeException("还没有设置Node 节点");
        }
        while (true) {
            try {

                localNode.decrementBalance();

                byte[] payload = JSONObject.toJSONBytes(localNode, SerializerFeature.WriteEnumUsingToString);
                client.setData().forPath(pathRegistered, payload);
                return true;
            } catch (Exception e) {
                return false;
            }
        }

    }

    /**
     * 创建父节点
     * @param managePath 父节点路径
     */
    private void createParentIfNeeded(String managePath) {
        try {
            Stat stat = client.checkExists().forPath(managePath);
            if (null == stat) {
                client.create().creatingParentsIfNeeded().withProtection().withMode(CreateMode.PERSISTENT)
                        .forPath(managePath);
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    /**
     * 返回本地的节点信息
     * @return 本地的节点信息
     */
    public ImNode getLocalNodeInfo() {
        return localNode;
    }
}

import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.recipes.cache.ChildData;
import org.apache.curator.framework.recipes.cache.PathChildrenCache;
import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent;
import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.alibaba.fastjson.JSONObject;
import com.test.config.ZookeeperConfig;

/**
 * 节点管理
 * @author 程就人生
 * @date 2020年1月13日
 */
public class PeerManager {
    
    private static Logger log = LoggerFactory.getLogger(PeerManager.class);
    
    private CuratorFramework client;
    
    private String managerPath;
    
    private static PeerManager singleInstance = null;
    
    /**
     * 获取单例
     * @return
     */
    public static PeerManager getInst() {
        if (null == singleInstance) {
            singleInstance = new PeerManager();         
            singleInstance.managerPath = ZookeeperConfig.managerPath;
            singleInstance.client = ZookeeperConfig.createInstance();
            singleInstance.client.start();
        }
        return singleInstance;
    }

    /**
     * 私有的构造方法
     */
    private PeerManager() {

    }

    /**
     * 初始化节点管理
     * 订阅节点的增加和删除事件绑定
     */
    @SuppressWarnings("resource")
    public void init() {
        try {
            // 订阅节点的增加和删除事件
            PathChildrenCache childrenCache = new PathChildrenCache(client, managerPath, true);
            
            PathChildrenCacheListener childrenCacheListener = new PathChildrenCacheListener() {

                @Override
                public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception {
                    log.info("开始监听其他的ImWorker子节点:-----");
                    ChildData data = event.getData();
                    switch (event.getType()) {
                    case CHILD_ADDED:
                        log.info("CHILD_ADDED : " + data.getPath() + "  数据:" + data.getData());
                        processNodeAdded(data);
                        break;
                    case CHILD_REMOVED:
                        log.info("CHILD_REMOVED : " + data.getPath() + "  数据:" + data.getData());
                        processNodeRemoved(data);
                        break;
                    case CHILD_UPDATED:
                        log.info("CHILD_UPDATED : " + data.getPath() + "  数据:" + new String(data.getData()));
                        break;
                    default:
                        log.debug("[PathChildrenCache]节点数据为空, path={}" + (data == null ? "null" : data.getPath()));
                        break;
                    }

                }

            };

            childrenCache.getListenable().addListener(childrenCacheListener);
            
            log.info("Register zk watcher successfully!");
            
            childrenCache.start(PathChildrenCache.StartMode.POST_INITIALIZED_EVENT);

        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    /**
     * 节点删除
     * @param data
     */
    private void processNodeRemoved(ChildData data) {

        byte[] payload = data.getData();
        
        ImNode n = JSONObject.parseObject(payload, ImNode.class);

        long id = ImWorker.getInst().getIdByPath(data.getPath());
        
        n.setId(id);
        
        log.info("[TreeCache]节点删除, path={}, data={}",data.getPath(),JSONObject.toJSONString(n));
        
    }

    /**
     * 节点增加
     * @param data
     */
    private void processNodeAdded(ChildData data) {
        
        byte[] payload = data.getData();
        
        ImNode n = JSONObject.parseObject(payload, ImNode.class);

        long id = ImWorker.getInst().getIdByPath(data.getPath());
        
        n.setId(id);

        log.info("[TreeCache]节点更新端口, path={}, data={}",data.getPath(),JSONObject.toJSONString(n));

        if (n.equals(getLocalNode())) {
            log.info("[TreeCache]本地节点, path={}, data={}",data.getPath(),JSONObject.toJSONString(n));
            return;
        }
    }
    

    /**
     * 获取本地节点
     * @return
     */
    public ImNode getLocalNode() {
        return ImWorker.getInst().getLocalNodeInfo();
    }

    /**
     * 移除节点
     * @param remoteNode
     */
    public void remove(ImNode remoteNode) {
        log.info("[TreeCache]移除远程节点信息,  node={}"+JSONObject.toJSONString(remoteNode));
    }
}

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

import org.apache.curator.framework.CuratorFramework;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.parser.Feature;

/**
 * 负载均衡节点获取
 * @author 程就人生
 * @date 2020年1月13日
 */
@Component
public class ImLoadBalance {
    
    private static Logger log = LoggerFactory.getLogger(ImLoadBalance.class);

    @Value("${zookeeper.register.node}")
    private String managerPath;
    
    @Autowired
    private CuratorFramework zkClient;

    /**
     * 获取负载最小的IM节点
     *
     * @return
     */
    public ImNode getBestWorker() {
        List<ImNode> workers = getWorkers();
        log.info("全部节点如下:");
        workers.stream().forEach(node -> {
            log.info("节点信息:{}",JSONObject.toJSONString(node));
        });
        ImNode best = balance(workers);
        return best;
    }

    /**
     * 按照负载排序
     *
     * @param items 所有的节点
     * @return 负载最小的IM节点
     */
    protected ImNode balance(List<ImNode> items) {
        if (items.size() > 0) {
            // 根据balance值由小到大排序
            Collections.sort(items);

            // 返回balance值最小的那个
            ImNode node = items.get(0);

            log.info("最佳的节点为:{}",JSONObject.toJSONString(node));
            return node;
        } else {
            return null;
        }
    }


    /**
     * 从zookeeper中拿到所有IM节点
     */
    protected List<ImNode> getWorkers() {

        List<ImNode> workers = new ArrayList<ImNode>();

        List<String> children = null;
        try {
            children = zkClient.getChildren().forPath(managerPath);
        } catch (Exception e) {
            e.printStackTrace();
            return workers;
        }

        for (String child : children) {
            log.info("child:"+child);
            byte[] payload = null;
            try {
                payload = zkClient.getData().forPath(managerPath+"/"+child);

            } catch (Exception e) {
                e.printStackTrace();
            }
            if (null == payload) {
                continue;
            }
            ImNode worker = JSONObject.parseObject(payload, ImNode.class, Feature.AllowArbitraryCommas);
            workers.add(worker);
        }
        return workers;

    }
}

第五,启动文件;

import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;

import com.test.distributed.ImWorker;
import com.test.distributed.PeerManager;

/**
 * zookeeper集群测试
 * @author 程就人生
 * @date 2020年1月13日
 */
@SpringBootApplication
public class ZookeeperDemo1Application implements CommandLineRunner{

    @Value("${server.port}")
    private int port;
        
    public static void main(String[] args) {        
        //获取application的上下文
        SpringApplication.run(ZookeeperDemo1Application.class, args);
    }

    /**
     * 项目启动后,将节点加入到zookeeper,保证一个服务器一个节点
     */
    @Override
    public void run(String... args) throws Exception {
        //组装节点信息
        ImWorker.getInst().setLocalNode("localhost", port);
        //节点初始化
        ImWorker.getInst().init();
        //启动节点的管理,节点之间的通信
        PeerManager.getInst().init(); 
    }

}

第六,测试文件;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RestController;

import com.test.distributed.ImLoadBalance;
import com.test.distributed.ImWorker;

/**
 * 节点测试
 * @author 程就人生
 * @date 2020年1月13日
 */
@RestController
public class IndexController {
    
    @Autowired
    private ImLoadBalance imLoadBalance;
    
    /**
     * 增加访问数量
     * @return
     */
    @GetMapping("/add")
    public Object addVisit(){
        return ImWorker.getInst().incBalance();
    }
    
    /**
     * 减少访问数量
     * @return
     */
    @GetMapping("/delete")
    public Object deleVisit(){
        return ImWorker.getInst().decrBalance();
    }
    
    /**
     * 获取访问量最少的节点
     * @return
     *
     */
    @GetMapping("/node")
    public Object getNodes(){
        return imLoadBalance.getBestWorker();
    }
}

最后,测试
先把zookeeper的三个服务器启动起来,然后,分别以端口号8400和8401启动项目,最后在浏览器地址栏中输入地址;

增加节点

获取访问量最少的节点

通过zkCli.cmd查看节点信息;
节点信息查看

在本demo中,代码简洁了很多,业务逻辑也足够清晰;但是配置的获取还是很麻烦,需要一个一个单独地获取,就不能自动配置了吗?答案是可以的,springboot也集成了zookeeper,实现了自动配置,具体如何使用,还等下回demo。