image.png
filebeat收集日志同步到redis或kafka，轻量级！！
下载方式
1，安装filebeat
[root@\ es02~]# wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.12.1-x86_64.rpm
[root@\ es02~]# yum install -y filebeat-7.12.1-x86_64.rpm
# 查看配置文件
[root@\ es02~]# grep -Ev '#' /etc/filebeat/filebeat.yml | grep -E '[^\ ]'
filebeat.inputs:
- type: log
enabled: false
paths:
- /var/log/*.log
- type: filestream
enabled: false
paths:
- /var/log/*.log
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
setup.template.settings:
index.number_of_shards: 1
setup.kibana:
output.elasticsearch:
hosts: ["localhost:9200"]
processors:
- add_host_metadata:
when.not.contains.tags: forwarded
- add_cloud_metadata: ~
- add_docker_metadata: ~
- add_kubernetes_metadata: ~
使用文档
2，自定义配置文件，收集文件日志，输出到文件中
# 原先的配置文件做备份
[root@\ es02~]# mv /etc/filebeat/filebeat.yml /etc/filebeat/filebeat.yml.bak
# 编写配置文件（输出到文件中）
[root@\ es02~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs: # filebeat输入
- type: log
paths:
- /var/log/messages
- /var/log/*.log
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
setup.template.settings:
index.number_of_shards: 1
setup.kibana:
output.file: # filebeat输出
path: "/tmp/filebeat"
filename: filebeat.log
processors:
- add_host_metadata:
when.not.contains.tags: forwarded
- add_cloud_metadata: ~
- add_docker_metadata: ~
- add_kubernetes_metadata: ~
# 启动配置文件
[root@\ es02~]# /usr/bin/filebeat -c /etc/filebeat/filebeat.yml
helli
hello
或者
[root@\ es02~]# systemctl start filebeat.service
# 查看日志文件
[root@\ es02/tmp/filebeat]# ll
total 9152
-rw------- 1 root root 9371024 May 13 16:34 filebeat.log
[root@\ es02/tmp/filebeat]# cat filebeat.log
... ...
{"@timestamp":"2021-05-13T08:34:02.914Z","@metadata":{"beat":"filebeat","type":"_doc","version":"7.12.1"},"log":{"offset":768680,"file":{"path":"/var/log/messages"}},"message":"May 13 16:33:57 es02 systemd-logind: New session 1830 of user root.","input":{"type":"log"},"ecs":{"version":"1.8.0"},"host":{"os":{"platform":"centos","version":"7 (Core)","family":"redhat","name":"CentOS Linux","kernel":"3.10.0-1160.21.1.el7.x86_64","codename":"Core","type":"linux"},"id":"18321d7070024b2cbb8a9d8132640345","containerized":false,"name":"es02","ip":["192.168.15.71","fe80::20c:29ff:fe43:5179","172.16.1.71","fe80::20c:29ff:fe43:5183"],"mac":["00:0c:29:43:51:79","00:0c:29:43:51:83"],"hostname":"es02","architecture":"x86_64"},"agent":{"version":"7.12.1","hostname":"es02","ephemeral_id":"9eb5d3bc-e88b-4fd4-bf89-a14843c3dd48","id":"bc9227b9-4e96-4380-baa8-6e3ab92e6f41","name":"es02","type":"filebeat"}}
把日志内容复制到json里查看
如下图。图1
图1
3，收集日志，输出到redis
[root@\ es02~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs: # filebeat输入
- type: log
paths:
- /var/log/messages
- /var/log/*.log
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
setup.template.settings:
index.number_of_shards: 1
setup.kibana:
output.redis:
hosts: ["192.168.15.71"]
password: ""
key: "filebeat"
db: 0
timeout: 5
processors:
- add_host_metadata:
when.not.contains.tags: forwarded
- add_cloud_metadata: ~
- add_docker_metadata: ~
- add_kubernetes_metadata: ~
# 修改redis配置文件监听端口
[root@\ es02~]# vim /etc/redis
bind 0.0.0.0
# 启动redis
[root@\ es02~]# systemctl restart redis
# 重启filebeat
[root@\ es02~]# systemctl restart filebeat.service
# 进入redis查看
127.0.0.1:6379> keys *
1) "filebeat"
4, 从reids输出到Elasticsearch
查看文档
https://www.elastic.co/guide/en/logstash/current/plugins-inputs-redis.html
[root@\ es02~]# vim redis-system.conf
input {
redis {
data_type => 'list'
host => "192.168.15.71"
key => "filebeat"
port => 6379
}
}
output {
elasticsearch {
hosts => ["172.16.1.70:9200"]
index => "filebeat-systemlog-%{+YYYY.MM.dd}"
}
}
[root@es-02 ~]# /usr/share/logstash/bin/logstash -f redis-system.conf
查看，见下图，图2，图3
图2
图3
5,关联kibana
PS：
ES02内存不足，把redis-system.conf卸载在es02上，
重新执行/usr/share/logstash/bin/logstash -f redis-system.conf
查看日志
6，nginx反向代理
[root@\ es01~]# yum install -y nginx
[root@\ es01~]# vim /etc/nginx/conf.d/kibana.conf
server {
listen 80;
server_name kibana.default.cluster.local.com;
location / {
proxy_pass http://172.16.1.71:5601;
}
[root@\ es01~]# nginx -t
[root@\ es01~]# systemctl restart nginx
域名登录
创建索引
查看日志
日志测试
注意：filebeat下载在es02机器上，并做了配置，所以在这台机器上测试
[root@\ es02~]# cd /var/log/
[root@\ es02/var/log]# echo 'cute baby' >> messages
查看日志如下图，图6
图6
nginx优化kibana
[root@\ es01~]# yum install -y httpd-tools
[root@\ es01~]# cd /etc/nginx/
[root@\ es01/etc/nginx]# htpasswd -c auth kibana
New password:
Re-type new password:
Adding password for user kibana
# 密码为123456
#优化nginx配置
[root@\ es01~]# vim /etc/nginx/conf.d/kibana.conf
server {
listen 80;
server_name kibana.nginx.com;
auth_basic "User Authentication";
auth_basic_user_file /etc/nginx/auth;
location / {
proxy_pass http://172.16.1.71:5601;
}
}
server {
listen 80 default_server;
server_name locahost;
return 302 return 302 http://kibana.nginx.com/;
}
[root@\ es01~]# systemctl restart nginx
再次登录测试，如下图，图7，图8
这个时候用ip依然可以访问，如图9
图7
图8
图9
nginx继续优化kibana
# 修改kibana配置文件，只能监听内网段
[root@\ es02~]# vim /etc/kibana/kibana.yml
server.host: "172.16.1.71"
[root@\ es01~]# vim /etc/nginx/conf.d/kibana.conf
server {
listen 80;
server_name kibana.nginx.com;
auth_basic "User Authentication";
auth_basic_user_file /etc/nginx/auth;
location / {
proxy_pass http://172.16.1.71:5601;
}
}
server {
listen 80 default_server;
server_name locahost;
return 302 http://kibana.nginx.com/;
}
# 重启kibana和nginx
[root@\ es02~]# systemctl restart kibana.service
[root@\ es01~]# systemctl restart nginx
测试外网ip无法访问 见图10
图10
7，监控
搜索metric接口
prometheus
官网
# 监控的机器改配置，增加以下内容
[root@\ prometheus~]# vim prometheus.yml
...
- job_name: "ELK"
static_configs:
- targets: ["192.168.15.71:9114"]
# 部署elasticsearch export
[root@\ es02~]# docker run --rm -p 9114:9114 -e "--es.uri=http://172.16.1.70:9200/" justwatch/elasticsearch_exporter:1.1.0
已创建metric接口
查看已关联elk
测试查询监控项
