0.配置加载
配置文件为keyFile形式，内容为key=value，示例如下：
[default]
pcapReadMethod=tpacketv3
tpacketv3BlockSize=2097152
interface=eth0
tpacketv3NumThreads=2
读入流程为，读取配置文件优先级高于其他所有事项优先级
flowchart LR
main --> moloch_config_init-->moloch_config_load
配置文件名存在 config.configfile中，读入的配置信息在哈希表config.DontSaveTags中
MolochConfig_t config;
typedef struct moloch_config {
// .....
char * configfile
// .....
}
moloch_config_load将keyfile内容读入全局变量molochKeyFile
void moloch_config_load()
{
keyfile = molochKeyFile = g_key_file_new();
status = g_key_file_load_from_file(keyfile, config.configFile, G_KEY_FILE_NONE, &error);
}
moloch_config_str将key对应的value从molochKeyFile中取出
gchar *moloch_config_str(GKeyFile *keyfile, char *key, char *d)
{
char *result;
if (!keyfile)
keyfile = molochKeyFile;
if (config.override && keyfile == molochKeyFile && (result = g_hash_table_lookup(config.override, key))) {
if (result[0] == 0)
result = NULL;
else
result = g_strdup(result);
} else if (g_key_file_has_key(keyfile, config.nodeName, key, NULL)) {
result = g_key_file_get_string(keyfile, config.nodeName, key, NULL);
} else if (config.nodeClass && g_key_file_has_key(keyfile, config.nodeClass, key, NULL)) {
result = g_key_file_get_string(keyfile, config.nodeClass, key, NULL);
} else if (g_key_file_has_key(keyfile, "default", key, NULL)) {
result = g_key_file_get_string(keyfile, "default", key, NULL);
} else if (d) {
result = g_strdup(d);
} else {
result = NULL;
}
if (result)
g_strstrip(result);
if (config.debug) {
LOG("%s=%s", key, result?result:"(null)");
}
return result;
}
简略配置存取流程图如下
image.png
1. Read流程分析
- 注册
Arkime默认采用libpcap经过内核七层协议栈解析的reader_libpcap_init方法，这里分析的流程是采用Arkime官方文档推荐的reader_tpacketv3_init方法，搭配pf_ring插件可以绕过内核协议栈解析提高抓包效率，无论采用何种抓包库，Arkime Reader整体注册和调用都是一样的。
- 调用
infos数组定义如下，i, j两个维度分别对应网口、线程，用于对不同网口启用多个线程进行流量收集，infos数组由reader_tpacketv3_init初始化，记录了抓包的配置信息，例如BlockSize等，infos收集流量分组后经过 reader_tpacketv3_thread整理成包交给packet线程进一步处理。
LOCAL MolochTPacketV3_t infos[MAX_INTERFACES][MAX_TPACKETV3_THREADS];
reader_tpacketv3_init方法进行了如下动作：
1.对infos数组进行初始化和配置文件赋值
2.**将infos元素中的fd绑定一个socket描述符，并mmap到infos[i][t].map中 **
3.将moloch_reader_start赋值为reader_tpacketv3_start
void reader_tpacketv3_init(char *UNUSED(name))
{
int blocksize = moloch_config_int(NULL, "tpacketv3BlockSize", 1<<21, 1<<16, 1U<<31);
numThreads = moloch_config_int(NULL, "tpacketv3NumThreads", 2, 1, MAX_TPACKETV3_THREADS);
if (blocksize % getpagesize() != 0) {
CONFIGEXIT("tpacketv3BlockSize=%d not divisible by pagesize %d", blocksize, getpagesize());
}
if (blocksize % config.snapLen != 0) {
CONFIGEXIT("tpacketv3BlockSize=%d not divisible by snapLen=%u", blocksize, config.snapLen);
}
moloch_packet_set_dltsnap(DLT_EN10MB, config.snapLen);
pcap_t *dpcap = pcap_open_dead(pcapFileHeader.dlt, pcapFileHeader.snaplen);
if (config.bpf) {
if (pcap_compile(dpcap, &bpf, config.bpf, 1, PCAP_NETMASK_UNKNOWN) == -1) {
CONFIGEXIT("Couldn't compile bpf filter: '%s' with %s", config.bpf, pcap_geterr(dpcap));
}
}
int fanout_group_id = moloch_config_int(NULL, "tpacketv3ClusterId", 8005, 0x0001, 0xffff);
int version = TPACKET_V3;
int i;
for (i = 0; i < MAX_INTERFACES && config.interface[i]; i++) {
int ifindex = if_nametoindex(config.interface[i]);
for (int t = 0; t < numThreads; t++) {
infos[i][t].fd = socket(AF_PACKET, SOCK_RAW, 0);
infos[i][t].interfacePos = i;
if (setsockopt(infos[i][t].fd, SOL_PACKET, PACKET_VERSION, &version, sizeof(version)) < 0)
CONFIGEXIT("Error setting TPACKET_V3, might need a newer kernel: %s", strerror(errno));
memset(&infos[i][t].req, 0, sizeof(infos[i][t].req));
infos[i][t].req.tp_block_size = blocksize;
infos[i][t].req.tp_block_nr = 64;
infos[i][t].req.tp_frame_size = config.snapLen;
infos[i][t].req.tp_frame_nr = (blocksize * infos[i][t].req.tp_block_nr) / infos[i][t].req.tp_frame_size;
infos[i][t].req.tp_retire_blk_tov = 60;
infos[i][t].req.tp_feature_req_word = 0;
if (setsockopt(infos[i][t].fd, SOL_PACKET, PACKET_RX_RING, &infos[i][t].req, sizeof(infos[i][t].req)) < 0)
CONFIGEXIT("Error setting PACKET_RX_RING: %s", strerror(errno));
struct packet_mreq mreq;
memset(&mreq, 0, sizeof(mreq));
mreq.mr_ifindex = ifindex;
mreq.mr_type = PACKET_MR_PROMISC;
if (setsockopt(infos[i][t].fd, SOL_PACKET, PACKET_ADD_MEMBERSHIP, &mreq, sizeof(mreq)) < 0)
CONFIGEXIT("Error setting PROMISC: %s", strerror(errno));
if (config.bpf) {
struct sock_fprog fcode;
fcode.len = bpf.bf_len;
fcode.filter = (struct sock_filter *)bpf.bf_insns;
if (setsockopt(infos[i][t].fd, SOL_SOCKET, SO_ATTACH_FILTER, &fcode, sizeof(fcode)) < 0)
CONFIGEXIT("Error setting SO_ATTACH_FILTER: %s", strerror(errno));
}
infos[i][t].map = mmap64(NULL, infos[i][t].req.tp_block_size * infos[i][t].req.tp_block_nr,
PROT_READ | PROT_WRITE, MAP_SHARED | MAP_LOCKED, infos[i][t].fd, 0);
if (unlikely(infos[i][t].map == MAP_FAILED)) {
CONFIGEXIT("MMap64 failure in reader_tpacketv3_init, %d: %s. Tried to allocate %d bytes (tpacketv3BlockSize: %d * 64) which was probbaly too large for this host, you probably need to reduce one of the values.", errno, strerror(errno), infos[i][t].req.tp_block_size * infos[i][t].req.tp_block_nr, blocksize);
}
infos[i][t].rd = malloc(infos[i][t].req.tp_block_nr * sizeof(struct iovec));
uint16_t j;
for (j = 0; j < infos[i][t].req.tp_block_nr; j++) {
infos[i][t].rd[j].iov_base = infos[i][t].map + (j * infos[i][t].req.tp_block_size);
infos[i][t].rd[j].iov_len = infos[i][t].req.tp_block_size;
}
struct sockaddr_ll ll;
memset(&ll, 0, sizeof(ll));
ll.sll_family = PF_PACKET;
ll.sll_protocol = htons(ETH_P_ALL);
ll.sll_ifindex = ifindex;
if (bind(infos[i][t].fd, (struct sockaddr *) &ll, sizeof(ll)) < 0)
CONFIGEXIT("Error binding %s: %s", config.interface[i], strerror(errno));
int fanout_type = PACKET_FANOUT_HASH;
int fanout_arg = ((fanout_group_id+i) | (fanout_type << 16));
if(setsockopt(infos[i][t].fd, SOL_PACKET, PACKET_FANOUT, &fanout_arg, sizeof(fanout_arg)) < 0)
CONFIGEXIT("Error setting packet fanout parameters: tpacketv3ClusterId: %d (%s)", fanout_group_id, strerror(errno));
}
fanout_group_id++;
}
pcap_close(dpcap);
if (i == MAX_INTERFACES) {
CONFIGEXIT("Only support up to %d interfaces", MAX_INTERFACES);
}
moloch_reader_start = reader_tpacketv3_start;
moloch_reader_exit = reader_tpacketv3_exit;
moloch_reader_stats = reader_tpacketv3_stats;
}
reader_tpacketv3_start起多线程抓包
void reader_tpacketv3_start() {
char name[100];
for (int i = 0; i < MAX_INTERFACES && config.interface[i]; i++) {
for (int t = 0; t < numThreads; t++) {
snprintf(name, sizeof(name), "moloch-af3%d-%d", i, t);
g_thread_unref(g_thread_new(name, &reader_tpacketv3_thread, &infos[i][t]));
}
}
}
reader_tpacketv3_thread监听socket描述符并收集分组组成完整packet
LOCAL void *reader_tpacketv3_thread(gpointer infov)
{
MolochTPacketV3_t *info = (MolochTPacketV3_t *)infov;
struct pollfd pfd;
int pos = 0;
memset(&pfd, 0, sizeof(pfd));
pfd.fd = info->fd;
pfd.events = POLLIN | POLLERR;
pfd.revents = 0;
// 初始化处理集合
MolochPacketBatch_t batch;
moloch_packet_batch_init(&batch);
// 当收到SIGINT信号时触发config.quitting
while (!config.quitting) {
// 拿到当前packet块指针
struct tpacket_block_desc *tbd = info->rd[pos].iov_base;
if (config.debug > 2) {
int i;
int cnt = 0;
int waiting = 0;
for (i = 0; i < (int)info->req.tp_block_nr; i++) {
struct tpacket_block_desc *stbd = info->rd[i].iov_base;
if (stbd->hdr.bh1.block_status & TP_STATUS_USER) {
cnt++;
waiting += stbd->hdr.bh1.num_pkts;
}
}
LOG("Stats pos:%d info:%d status:%x waiting:%d total cnt:%d total waiting:%d", pos, info->interfacePos, tbd->hdr.bh1.block_status, tbd->hdr.bh1.num_pkts, cnt, waiting);
}
// Wait until the block is owned by moloch | poll轮询等待文件描述符状态改变(等待包写入信息)
if ((tbd->hdr.bh1.block_status & TP_STATUS_USER) == 0) {
poll(&pfd, 1, -1);
continue;
}
struct tpacket3_hdr *th;
// 获取包头部
th = (struct tpacket3_hdr *) ((uint8_t *) tbd + tbd->hdr.bh1.offset_to_first_pkt);
uint16_t p;
// 根据包头部得到包数量，遍历处理
for (p = 0; p < tbd->hdr.bh1.num_pkts; p++) {
if (unlikely(th->tp_snaplen != th->tp_len)) {
LOGEXIT("ERROR - Arkime requires full packet captures caplen: %d pktlen: %d\n"
"See https://arkime.com/faq#moloch_requires_full_packet_captures_error",
th->tp_snaplen, th->tp_len);
}
// 对包内容进行初步解析，填充packet
MolochPacket_t *packet = MOLOCH_TYPE_ALLOC0(MolochPacket_t);
packet->pkt = (u_char *)th + th->tp_mac;
packet->pktlen = th->tp_len;
packet->ts.tv_sec = th->tp_sec;
packet->ts.tv_usec = th->tp_nsec/1000;
packet->readerPos = info->interfacePos;
if ((th->tp_status & TP_STATUS_VLAN_VALID) && th->hv1.tp_vlan_tci) {
packet->vlan = th->hv1.tp_vlan_tci & 0xfff;
}
// 进一步解析包内容, 填充packet
moloch_packet_batch(&batch, packet);
th = (struct tpacket3_hdr *) ((uint8_t *) th + th->tp_next_offset);
}
// 将处理过后的包入队packetQ交给packet线程继续处理
moloch_packet_batch_flush(&batch);
tbd->hdr.bh1.block_status = TP_STATUS_KERNEL;
pos = (pos + 1) % info->req.tp_block_nr;
}
return NULL;
}
到此为止read流程结束，捕获到的包进入packetQ双向链表队列等待packet thread处理