文章实时采集(日志解析失败错误json._root.yml)

　　文章实时采集(日志解析失败错误json._root.yml)

　　说明 Filebeat 版本为 5.3.0

　　之所以用beats家族的Filebeat来代替Logstash，是因为Logstash消耗资源太多（如果服务器资源充足请忽略）

　　官网下载Logstash有89M，而Filebeat只有8.4M，可见

　　Logstash 可以配置 jvm 参数。经过我自己调试，内存分配小，启动很慢，有时根本起不来。如果分配很大，其他服务将没有资源。

　　都说对于低配置的服务器，选择Filebeat是最好的选择，现在Filebeat已经开始替代Logstash了。还是需要修改nginx的日志格式，nginx.config。

　　更改日志记录的格式

　　 log_format json '{ "@timestamp": "$time_iso8601", '

'"time": "$time_iso8601", '

'"remote_addr": "$remote_addr", '

'"remote_user": "$remote_user", '

'"body_bytes_sent": "$body_bytes_sent", '

'"request_time": "$request_time", '

'"status": "$status", '

'"host": "$host", '

'"request": "$request", '

'"request_method": "$request_method", '

'"uri": "$uri", '

'"http_referrer": "$http_referer", '

'"body_bytes_sent":"$body_bytes_sent", '

'"http_x_forwarded_for": "$http_x_forwarded_for", '

'"http_user_agent": "$http_user_agent" '

'}';

access_log /var/log/nginx/access.log json;

　　文件节拍.yml

　　 #=========================== Filebeat prospectors =============================

filebeat.prospectors:

- input_type: log

# Paths that should be crawled and fetched. Glob based paths.

paths:

- /var/log/nginx/*access*.log

json.keys_under_root: true

json.overwrite_keys: true

#-------------------------- Elasticsearch output ------------------------------

output.elasticsearch:

# Array of hosts to connect to.

hosts: ["ip:port","ip:port"]

index: "filebeat_server_nginx_%{+YYYY-MM}"

　　这里要注意的是

　　json.keys_under_root：默认值为FALSE，表示我们的json日志解析后会放在json key上。设置为 TRUE，所有键都将放置在根节点中

　　json.overwrite_keys：是否覆盖原来的key，这个是key的配置。设置keys_under_root为TRUE后，再设置overwrite_keys为TRUE，覆盖filebeat的默认key值

　　还有其他配置

　　json.add_error_key: 添加json_error key key 记录json解析失败错误

　　json.message_key：指定解析后的json日志放哪个key，默认为json，也可以指定log等。

　　说白了，区别就是elasticsearch的配置前的数据是这样的：

　　 {

"_index": "filebeat_server_nginx_2018-05",

"_type": "log",

"_id": "AWM9sVOkCcRcg0IPg399",

"_version": 1,

"_score": 1,

"_source": {

"@timestamp": "2018-05-08T03:00:17.544Z",

"beat": {

"hostname": "VM_252_18_centos",

"name": "VM_252_18_centos",

"version": "5.3.0"

},

"input_type": "log",

"json": {},

"message": "{ "@timestamp": "2018-05-08T11:00:11+08:00", "time": "2018-05-08T11:00:11+08:00", "remote_addr": "113.16.251.67", "remote_user": "-", "body_bytes_sent": "403", "request_time": "0.000", "status": "200", "host": "blog.joylau.cn", "request": "GET /img/%E7%BD%91%E6%98%93%E4%BA%91%E9%9F%B3%E4%B9%90.png HTTP/1.1", "request_method": "GET", "uri": "/img/\xE7\xBD\x91\xE6\x98\x93\xE4\xBA\x91\xE9\x9F\xB3\xE4\xB9\x90.png", "http_referrer": "http://blog.joylau.cn/css/style.css", "body_bytes_sent":"403", "http_x_forwarded_for": "-", "http_user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36" }",

"offset": 7633,

"source": "/var/log/nginx/access.log",

"type": "log"

}

}

　　配置好之后是这样的：

　　 {

"_index": "filebeat_server_nginx_2018-05",

"_type": "log",

"_id": "AWM9rjLd8mVZNgvhdnN9",

"_version": 1,

"_score": 1,

"_source": {

"@timestamp": "2018-05-08T02:56:50.000Z",

"beat": {

"hostname": "VM_252_18_centos",

"name": "VM_252_18_centos",

"version": "5.3.0"

},

"body_bytes_sent": "12576",

"host": "blog.joylau.cn",

"http_referrer": "http://blog.joylau.cn/",

"http_user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36",

"http_x_forwarded_for": "-",

"input_type": "log",

"offset": 3916,

"remote_addr": "60.166.12.138",

"remote_user": "-",

"request": "GET /2018/03/01/JDK8-Stream-Distinct/ HTTP/1.1",

"request_method": "GET",

"request_time": "0.000",

"source": "/var/log/nginx/access.log",

"status": "200",

"time": "2018-05-08T10:56:50+08:00",

"type": "log",

"uri": "/2018/03/01/JDK8-Stream-Distinct/index.html"

}

}

　　所以看起来很舒服

　　启动 FileBeat

　　进入Filebeat目录

　　 nohup sudo ./filebeat -e -c filebeat.yml >/dev/null 2>&1 &

　　更新

　　如果 nginx 日志收录中文，则会将中文转换为 Unicode 编码。如果没有，只需添加 escape=json 参数。

　　 log_format json escape=json '{ "@timestamp": "$time_iso8601", '

'"time": "$time_iso8601", '

'"remote_addr": "$remote_addr", '

'"remote_user": "$remote_user", '

'"body_bytes_sent": "$body_bytes_sent", '

'"request_time": "$request_time", '

'"status": "$status", '

'"host": "$host", '

'"request": "$request", '

'"request_method": "$request_method", '

'"uri": "$uri", '

'"http_referrer": "$http_referer", '

'"body_bytes_sent":"$body_bytes_sent", '

'"http_x_forwarded_for": "$http_x_forwarded_for", '

'"http_user_agent": "$http_user_agent" '

'}';

access_log /var/log/nginx/access.log json;

　　信息