owt-server 的集群管理者、集群工作站、消息队列(三)

转载请注明:

原始地址:https://www.jianshu.com/p/5d77641b17b2

原作者:wonder


1、写在前面

前两篇(《owt-server 的集群管理者、集群工作站、消息队列(一)》、《owt-server 的集群管理者、集群工作站、消息队列(二)》)分别主要介绍了owt-server的 clusterManager 模块和 amqp_client 模块。clusterManager 模块是用于集群管理、调度的模块;amqp_client 模块是用于封装消息队列、为上层提供 rpc 接口的模块。

本文将介绍 clusterWorker 模块,该模块是集群工作站模块,clusterWorker是受 clusterManager 模块管理的对象。

除此以外,本篇还将:

1)介绍owt-server统计一个worker的硬件负载情况的策略

2)举一个栗子来说明一个clusterWorker的使用方式。

2、clusterWorker 模块分解

clusterWorker 模块位于owt-server源码目录下common/clusterWorker.js文件中。


var genID = (function() {    //产生随机ID的函数

    function s4() {

        return Math.floor((1 + Math.random()) * 0x10000)

                   .toString(16)

                   .substring(1);

    }

    return function() {

        return s4() + s4()

               /*+ '-' + s4()

               + '-' + s4()

               + '-' + s4()

               + '-'*/ + s4() + s4() + s4();

    };

})();

module.exports = function (spec) {    //导出内容

    var that = {};

    /*'unregistered' | 'registered' | 'recovering'*/    //状态类型

    var state = 'unregistered',

        tasks = [];

    var rpcClient = spec.rpcClient,        //rpc实例

        id = spec.purpose + '-' + genID() + '@' + (spec.info.hostname || spec.info.ip),    //该clusterWorker的ID

        purpose = spec.purpose,        //该clusterWorker承载的应用类型

        info = spec.info,    //相关配置

        cluster_name = spec.clusterName || 'owt-cluster',    //该clusterWorker的名称

        join_retry = spec.joinRetry || 60,    //加入cluster的重试次数

        keep_alive_period = 800/*MS*/,    //保活时长

        keep_alive_interval = undefined,    //保活间隔

        on_join_ok = spec.onJoinOK || function () {log.debug('Join cluster successfully.');},    //加入cluster成功时的回调函数

        on_join_failed = spec.onJoinFailed || function (reason) {log.debug('Join cluster failed. reason:', reason);},    //加入失败时的回调函数

        on_loss = spec.onLoss || function () {log.debug('Lost connection with cluster manager');},    //与cluster manager失联时的回调函数

        on_recovery = spec.onRecovery || function () {log.debug('Rejoin cluster successfully.');},    //重新加入cluster的回调函数

        on_overload = spec.onOverload || function () {log.debug('Overloaded!!');};;    //该clusterWorker过载时的回调函数

    var previous_load = 0.99;    //当前负载,初始为0.99

    var reportLoad = function (load) {    //报告当前worker负载情况到manager

        if (load == previous_load) {    //与上次负载相同,不报告

            return;

        }

        previous_load = load;

        if (state === 'registered') {

            rpcClient.remoteCast(    //向cluster发送负载报告

                cluster_name,

                'reportLoad',

                [id, load]);    

        }

        if (load > 0.98/*FIXME: Introduce a configuration item to specify the overload threshold here.*/) {

           on_overload();    //过载处理函数

        }

    };

    var load_collector = loadCollector({period: spec.loadCollection.period,     //负载统计间隔

                                        item: spec.loadCollection.item,    //指定负载收集依据(CPU、Network、GPU、Memory等)

                                        onLoad: reportLoad});    //创建负载收集器,并制定发送负载报告的函数

    var join = function (on_ok, on_failed) {    //执行一次加入cluster的rpc过程

        makeRPC(    //执行一次rpc

            rpcClient,    //rpc实例

            cluster_name,    // cluster的名称

            'join',

            [purpose, id, info],    // 消息包括:应用类型、该worker的id、配置信息

            function (result) {    //加入cluster成功的回调函数

                state = 'registered';    //设置worker当前状态为“registered”

                on_ok(result);

                previous_load = 0.99;    //设置初始负载

                keepAlive();    //执行保活函数

            }, on_failed);

    };

    var joinCluster = function (attempt) {    //加入cluster

        var countDown = attempt;    //尝试次数

        var tryJoin = function (countDown) {

            log.debug('Try joining cluster', cluster_name, ', retry count:', attempt - countDown);

            join(function (result) {    //执行 join 的rpc

                on_join_ok(id);

                log.info('Join cluster', cluster_name, 'OK.');

            }, function (error_reason) {     //加入失败的回调函数

                if (state === 'unregistered') {   

                    log.info('Join cluster', cluster_name, 'failed.');

                    if (countDown <= 0) {

                        log.error('Join cluster', cluster_name, 'failed. reason:', error_reason);

                        on_join_failed(error_reason);

                    } else {

                        tryJoin(countDown - 1);    //重试

                    }

                }

            });

        };

        tryJoin(attempt);    //尝试加入,尝试若干次

    };

    var keepAlive = function () {    //保活函数

        keep_alive_interval && clearInterval(keep_alive_interval);    //清空计时器

        var tryRecovery = function (on_success) {    //尝试恢复函数

            clearInterval(keep_alive_interval);    //清空计时

            keep_alive_interval = undefined;    

            state = 'recovering';    //状态置为“recovering”

            var tryJoining = function () {    //尝试加入

                log.debug('Try rejoining cluster', cluster_name, '....');

                join(function (result) {    //执行一次join rpc过程

                    log.debug('Rejoining result', result);

                    if (result === 'initializing') {    // manager是正处于始化状态

                        tasks.length > 0 && pickUpTasks(tasks);    //该worker向manager申请记录worker上的所有任务

                    } else {    //manager是其他状态

                        on_loss();    //worker掉线,处理掉线处理函数

                        tasks = [];    //任务置为空

                    }

                    on_success();    //与manager成功重连

                }, function (reason) {    //重连失败

                    if (state === 'recovering') {    //该worker处于重连状态

                        log.debug('Rejoin cluster', cluster_name, 'failed. reason:', reason);

                        tryJoining();    //再次尝试加入cluster

                    }

                });

            };

            tryJoining();    //开始尝试加入cluster

        };

        var loss_count = 0;

        keep_alive_interval = setInterval(function () {    //保活计时器启动

            makeRPC(    //执行保活rpc

                rpcClient,

                cluster_name,

                'keepAlive',

                [id],

                function (result) {

                    loss_count = 0;

                    if (result === 'whoareyou') {    //manager未记录该worker

                        if (state !== 'recovering') {    //该worker不处于“recovering”时,需重新加入

                            log.info('Unknown by cluster manager', cluster_name);

                            tryRecovery(function () {

                                log.info('Rejoin cluster', cluster_name, 'OK.');

                            });

                        }

                    }

                }, function (error_reason) {    //保活失败

                    loss_count += 1;    

                    if (loss_count > 3) {    //保活失败超过3次

                        if (state !== 'recovering') {    //该worker不处于“recovering”时,需重新加入

                            log.info('Lost connection with cluster', cluster_name);

                            tryRecovery(function () {

                                log.info('Rejoin cluster', cluster_name, 'OK.');

                                on_recovery(id);

                            });

                        }

                    }

                });

        }, keep_alive_period);    //保活间隔

    };

    var pickUpTasks = function (taskList) {    //通知manager:该worker正在执行某些任务

        rpcClient.remoteCast(

            cluster_name,

            'pickUpTasks',

            [id, taskList]);

    };

    var layDownTask = function (task) {     //通知manager:该worker结束某些任务

        rpcClient.remoteCast(

            cluster_name,

            'layDownTask',

            [id, task]);

    };

    var doRejectTask = function (task) {    //通知manager:该worker拒绝某些任务

        rpcClient.remoteCast(

            cluster_name,

            'unschedule',

            [id, task]);

    };

    that.quit = function () {    //导出函数:退出

        if (state === 'registered') {    

            if (keep_alive_interval) {

                clearInterval(keep_alive_interval);    //清理保活计时器

                keep_alive_interval = undefined;

            }

            rpcClient.remoteCast(    //通知manager:该worker退出

                cluster_name,

                'quit',

                [id]);

        } else if (state === 'recovering') {

            keep_alive_interval && clearInterval(keep_alive_interval);    //清理保活计时器

        }

        load_collector && load_collector.stop();    //停止该worker的负载统计

    };

    that.reportState = function (st) {    //导出函数:状态汇报

        if (state === 'registered') {    

            rpcClient.remoteCast(    //通知mananger:该worker的状态

                cluster_name,

                'reportState',

                [id, st]);

        }

    };

    that.addTask = function (task) {    //导出函数:添加任务

        var i = tasks.indexOf(task);    

        if (i === -1) {    //向该worker任务列表增加新任务

            tasks.push(task);

            if (state === 'registered') {    //向manager通知开始执该任务

                pickUpTasks([task]);

            }

        }

    };

    that.removeTask = function (task) {    //导出函数:移除某任务

        var i = tasks.indexOf(task);

        if (i !== -1) {    //从worker任务列表中移除该任务

            tasks.splice(i, 1);

            if (state === 'registered') {    //向manager通知移除该任务

                layDownTask(task);

            }

        }

    };

    that.rejectTask = function (task) {    //拒绝该任务

        doRejectTask(task);    //向manager通知,拒绝该任务

    };

    joinCluster(join_retry);    //开始加入cluster集群

    return that;

};


3、loadCollector模块分解

在第2节中,clusterWorker 需要向manager 汇报自身硬件的负载情况。这里就对 负载统计模块(loadCollector )进行分解,以形成一个直观的认识。该模块位于owt-server源码目录下的common/loadCollector.js文件


var child_process = require('child_process');    //引用 node.js 的子进程模块

var os = require('os');    //引用 node.js 的操作系统模块

var cpuCollector = function (period, onLoad) {    //基于CPU进行负载统计

    var olds = os.cpus();    //初始时,获取逻辑CPU内核的信息列表

    var begin = 0;

    var end = olds.length - 1;

    var interval = setInterval(function() {    //启动计时器

        var cpus = os.cpus();    //获取当前逻辑CPU内核的信息列表

        var idle = 0;

        var total = 0;

        for (let i = begin; i <= end; i++) {    //逐个CPU计算其负载

            for (let key in cpus[i].times) {    //统计没在模式下CPU花费的ms数,共[user , nice , sys, idle, irq ] 5种模式

                let diff = cpus[i].times[key] - olds[i].times[key];

                if (key === 'idle') {    //累计该CPU在 “空闲” 状态下的耗时

                    idle += diff;

                }

                total += diff;    //累计该CPU该时段的总耗时

            }

        }

        olds = cpus;    //记录当前各CPU状态

        onLoad(1 - idle/total);    //使用所有CPU的 “1-空闲比” 作为当前负载情况进行汇报

        log.debug('cpu usage:', 1 - idle/total);

    }, period);    //计时器触发间隔

    this.stop = function () {    //停止统计

        log.debug("To stop cpu load collector.");

        clearInterval(interval);

    };

};

var memCollector = function (period, onLoad) {    //基于Memory进行负载统计

    var interval = setInterval(function() {    //启动计时器

        var usage = 1 - os.freemem() / os.totalmem();    //内存负载为: 1-空闲比 

        onLoad(usage);    //汇报Memeory负载

        log.debug('mem usage:', usage);

    }, period);

    this.stop = function () {    //停止计时器

        log.debug("To mem cpu load collector.");

        clearInterval(interval);

    };

};

var diskCollector = function (period, drive, on_load) {    //基于 Disk 进行负载统计

    var interval = setInterval(function () {

        var total = 1, free = 0;

        child_process.exec("df -k '" + drive.replace(/'/g,"'\\''") + "'", function(err, stdout, stderr) {    //在指定目录执行 df -k

            if (err) {

                log.error(stderr);

            } else {

                var lines = stdout.trim().split('\n');    

                var str_disk_info = lines[lines.length - 1].replace( /[\s\n\r]+/g,' ');   

                var disk_info = str_disk_info.split(' ');  

                total = disk_info[1];   

                free = disk_info[3];

                on_load(Math.round((1.0 - free / total) * 1000) / 1000);    //汇报

            }

        });

    }, period);

    this.stop = function () {

        log.debug("To stop disk load collector.");

        clearInterval(interval);

    };

};

var networkCollector = function (period, interf, max_scale, on_load) {    //使用 Network 进行负载统计

    var rx_Mbps = 0, tx_Mbps = 0, rx_bytes = 0, tx_bytes = 0;

    var meter = setInterval(function () {    //启动定时器,1s统计一下网络收发情况

        child_process.exec("awk 'NR>2{if (index($1, \"" + interf + "\")==1){print $2, $10}}' /proc/net/dev", function (err, stdout, stderr) {    //在 网口interf 上执行该awk语句,从proc/net/dev抽取出“接收bytes” 和 “发送bytes”

            if (err) {

                log.error(stderr);

            } else {

                var fields = stdout.trim().split(" ");

                if (fields.length < 2) {

                    return log.warn('not ordinary network load data');

                }

                var rx = Number(fields[0]), tx = Number(fields[1]);    //分别赋值接收字节 和 发送字节

                if (rx >= rx_bytes && rx_bytes > 0) {    //转换单位为Mbps

                    rx_Mbps = Math.round(((rx - rx_bytes) * 8 / 1048576) * 1000) / 1000;

                }

                if (tx >= tx_bytes && tx_bytes > 0) {     //转换单位为Mbps

                    tx_Mbps = Math.round(((tx - tx_bytes) * 8 / 1048576) * 1000) / 1000;

                }

                rx_bytes = rx;

                tx_bytes = tx;

            }

        });

    }, 1000);     //1s统计一次

    var reporter = setInterval(function () {    //启动负载汇报定时器

        var rt_load = Math.round(Math.max(rx_Mbps / max_scale, tx_Mbps / max_scale) * 1000) / 1000;    //计算收发速率与基准速率比例

        on_load(rt_load);    //汇报负载

    }, period);    //汇报间隔

    this.stop = function () {    //停止统计

        log.debug("To stop network load collector.");

        meter && clearInterval(meter);

        reporter && clearInterval(reporter);

        meter = undefined;

        reporter = undefined;

    };

};

var gpuCollector = function (period, on_load) {    //基于 Gpu 进行负载统计

    var child = child_process.exec('stdbuf -o0 metrics_monitor 100 1000');    //这一条使用了子进程 metrics_monitor 进行资源统计

    var cpu_load = 0,

        cpu_collector = new cpuCollector(period, function (data) {cpu_load = data;});    //启用一个CPU负载统计

    var load = 0;

    child.stdout.on('data', function (data) {    //当上述命令得到结果时

        var usage_sum = 0, samples = 0;

        var lines = data.toString().split('\n');    

        var i = lines.length > 10 ? lines.length - 10 : 0;    

        for (; i < lines.length; i++) {    //处理每行

            var engine_list = lines[i].split('\t');

            var engine_max_usage = 0;

            for (var engine of engine_list) {    //处理每个引擎

                var m = null;

                if ((m = engine.match(/\s+usage:\s+(\d+\.\d+)/)) && m !== null && m.length > 1) {    //匹配到“usage字段”

                    var engine_usage = Number(m[1]);    //获取使用率

                    if (engine_max_usage < engine_usage)    //更新最大使用率

                        engine_max_usage = engine_usage;    

                }

            }

            usage_sum = usage_sum + engine_max_usage;    //累计最大使用率

            samples = samples + 1;    //采样数增加

        }

        if (samples > 0)

            load = (usage_sum / samples) / 100;    //平均最大使用率

        else

            load = 0;

    });

    var interval = setInterval(function () {    //启动汇报计时器

        var result = Math.max(load, cpu_load);        //报告CPU 和 GPU中最大的使用负载

        on_load(result);

    }, period);

    this.stop = function () {    //停止计时器

        log.debug("To stop gpu load collector.");

        cpu_collector && cpu_collector.stop();

        cpu_collector = undefined;

        child && child.kill();    //关闭子进程

        child = undefined;

        interval && clearInterval(interval);

        interval = undefined;

    };

};

exports.LoadCollector = function (spec) {    //导出负载统计模块

    var that = {};

    var period = spec.period || 1000,    //统计间隔

        item = spec.item,    //统计策略

        on_load = spec.onLoad || function (load) {log.debug('Got', item.name, 'load:', load);},    //回调函数

        collector = undefined;

    that.stop = function () {    //停止统计

        log.info("To stop load collector.");

        collector && collector.stop();

        collector = undefined;

    };

    switch (item.name) {

        case 'network':

            collector = new networkCollector(period, item.interf, item.max_scale, on_load);

            break;

        case 'cpu':

            collector = new cpuCollector(period, on_load);

            break;

        case 'gpu':

            collector = new gpuCollector(period, on_load);

            break;

        case 'memory':

            collector = new memCollector(period, on_load);

            break;

        case 'disk':

            collector = new diskCollector(period, item.drive, on_load);

            break;

        default:

            log.error('Unknown load item');

            return undefined;

            //break;

    }

    return that;

};


4、makeRPC 模块

第2节中出现了makeRPC函数,该模makeRPC块位于该模块位于owt-server源码目录下的common/makeRPC.js文件。仅仅是对amqp_client模块封装了一个响应处理函数,用于代处理“error” 、“timeout” 和 正常消息。


exports.makeRPC = function (rpcClient, remote_node, remote_function, parameters_list, on_ok, on_error) {

    rpcClient.remoteCall(

        remote_node,

        remote_function,

        parameters_list,

        {callback: function (result, error_reason) {

            if (result === 'error') {

                typeof on_error === 'function' && on_error(error_reason);

            } else if (result === 'timeout') {

                typeof on_error === 'function' && on_error('Timeout to make rpc to ' + remote_node + '.' + remote_function);

            } else {

                typeof on_ok === 'function' && on_ok(result);

            }

        }}

    );

};


5、makeRPC 模块