先设置nginx日志格式为josn
在nginx的http区域添加josn类型日志
log_format json_log escape=json '{"realip":"$remote_addr","@timestamp":"$time_iso8601","host":"$http_host","request":"$request","req_body":"$request_body","status":"$status","size":$body_bytes_sent,"ua":"$http_user_agent","request_method":"$request_method","domain":"$host","idc_tag":"fkidc","host":"$hostname","via":"$http_via","protocol":"$scheme","cookie":"$cookie_id","req_time":"$request_time","uri":"$uri","referer":"$http_referer","xff":"$http_x_forwarded_for","ups_status":"$upstream_status","ups_addr":"$upstream_addr","ups_time":"$upstream_response_time"}';
在server区域添加日志
logs/elk.log json_log;
docker的安装这里就不做介绍了，下面直接利用docker部署elk
下载elk镜像到本地这里使用的是sebp/elk
docker pull sebp/elk
docker run -p 5601:5601 -p 9200:9200 -p 5044:5044 -e ES_MIN_MEM=1024m -e ES_MAX_MEM=4096m -it --name elk sebp/elk
进入容器
docker exec -it elk /bin/bash
添加logstash的配置文件如下：
cd /etc/logstash/conf.d/
cat blog.iyunv.com_elk.conf
input {
file {
path => "/data/shuyuan/elk_blog.iyunv.com.log"
type => "nginx-access"
start_position => "beginning"
sincedb_path => "/dev/null"
codec => json
}
}
filter {
if [type] == "nginx-access" {
mutate {
split => [ "[ups_time]", "," ]
}
mutate {
convert => [ "[ups_time]", "float" ]
}
if [xff] != "-" {
geoip {
source => "[xff]"
target => "geoip"
database => "/opt/logstash/GeoLite2-City_20180327/GeoLite2-City.mmdb"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
mutate {
convert => [ "[geoip][coordinates]", "float" ]
split => [ "[xff]", "," ]
}
}
else if [realip] != "-" {
geoip {
source => "[realip]"
target => "geoip"
database => "/opt/logstash/GeoLite2-City_20180327/GeoLite2-City.mmdb"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
mutate {
convert => [ "[geoip][coordinates]", "float" ]
}
}
}
}
output {
if [type] == "nginx-access" {
elasticsearch {
hosts => ["localhost"]
index => "logstash-nginx-access-log-%{+YYYY-MM}"
workers => 1
manage_template => true
template_overwrite => true
}
}
}
上面的附件带格式
下载geoip数据放到/opt/logstash/
重新启动logstash
service logstash restart
外面访问kibana
下面放一张效果图：
