docker 部署elk以josn格式入库nginx日志

先设置nginx日志格式为josn

在nginx的http区域添加josn类型日志

log_format json_log escape=json '{"realip":"$remote_addr","@timestamp":"$time_iso8601","host":"$http_host","request":"$request","req_body":"$request_body","status":"$status","size":$body_bytes_sent,"ua":"$http_user_agent","request_method":"$request_method","domain":"$host","idc_tag":"fkidc","host":"$hostname","via":"$http_via","protocol":"$scheme","cookie":"$cookie_id","req_time":"$request_time","uri":"$uri","referer":"$http_referer","xff":"$http_x_forwarded_for","ups_status":"$upstream_status","ups_addr":"$upstream_addr","ups_time":"$upstream_response_time"}';

在server区域添加日志

logs/elk.log json_log;

docker的安装这里就不做介绍了，下面直接利用docker部署elk

下载elk镜像到本地这里使用的是sebp/elk

docker pull sebp/elk

docker run -p 5601:5601 -p 9200:9200 -p 5044:5044 -e ES_MIN_MEM=1024m -e ES_MAX_MEM=4096m -it --name elk sebp/elk

进入容器

docker exec -it elk /bin/bash

添加logstash的配置文件如下：

cd /etc/logstash/conf.d/

cat www.wangjingfeng.com_elk.conf

input {
file {
path => "/data/shuyuan/elk_www.wangjingfeng.com.log"
type => "nginx-access"
start_position => "beginning"
sincedb_path => "/dev/null"
codec => json
}
}
filter {
if [type] == "nginx-access" {
mutate {
split => [ "[ups_time]", "," ]
}
mutate {
convert => [ "[ups_time]", "float" ]
}

if [xff] != "-" {
geoip {
source => "[xff]"
target => "geoip"
database => "/opt/logstash/GeoLite2-City_20180327/GeoLite2-City.mmdb"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
mutate {
convert => [ "[geoip][coordinates]", "float" ]
split => [ "[xff]", "," ]
}
}
else if [realip] != "-" {
geoip {
source => "[realip]"
target => "geoip"
database => "/opt/logstash/GeoLite2-City_20180327/GeoLite2-City.mmdb"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
mutate {
convert => [ "[geoip][coordinates]", "float" ]
}

}
}
}
output {
if [type] == "nginx-access" {
elasticsearch {
hosts => ["localhost"]
index => "logstash-nginx-access-log-%{+YYYY-MM}"
workers => 1
manage_template => true
template_overwrite => true
}
}
}

www.wangjingfeng.com-nginx.conf

上面的附件带格式

下载geoip数据放到/opt/logstash/

GeoLite2-City_20180327.tar

重新启动logstash

service logstash restart

外面访问kibana

下面放一张效果图：