hadoop►logstash►logstash filter grok

logstash filter grok

logstash filter grok

  1. grok语法说明
    1
    2
    3
    4
    5
    6
    grok的语法格式为 %{SYNTAX:SEMANTIC}
    * SYNTAX是文本要匹配的模式
      例如: 3.14匹配NUMBER模式; 127.0.0.1匹配IP模式
    * SEMANTIC是匹配到的文本片段的标识
      例如: 3.14可以是一个时间的持续时间,所以可以简单地叫做"duration";字符串"55.3.244.1"可以被标识为"client"
      默认情况下,所有的SEMANTIC是以字符串的方式保存,如果想要转换一个SEMANTIC的数据类型:%{NUMBER:num:int}

log_format main ‘$remote_addr $remote_user [$time_local] “$request” ‘
‘$status $body_bytes_sent ‘
‘“$http_user_agent” $http_x_forwarded_for $request_time $upstream_response_time’;

101.201.57.207 - [11/May/2017:10:06:38 +0800] “POST /ssp-switch-automapping/ws/autoOnOff HTTP/1.1” 200 265 “Mozilla/4.0 (compatible; MSIE 6.0; MS Web Services Client Protocol 4.0.30319.42000)” - 0.301 0.297

%{IP:client} 101.201.57.207
%{WORD:method} POST

%{NUMBER:bytes} 265
%{NUMBER:duration} 0.301
%{NUMBER:duration} 0.297

[root@smallasa conf]# cat test.conf
input{
file {
path => [“/mnt/log/nginx/test/inbound.log”]
exclude => [“kibana.log”,”toolstmp.log”,”pennswitch*.log”]
type => “nginx_log”
discover_interval => 15
sincedb_path => “/mnt/data/logstash”
sincedb_write_interval => 15
stat_interval => 1
start_position => “beginning”
}
}
filter{

}
output{
stdout{
codec => rubydebug
}
}
[root@smallasa conf]# /mnt/app/logstash/bin/logstash -f /mnt/app/logstash/conf/test.conf –configtest
Configuration OK

[root@smallasa conf]# /mnt/app/logstash/bin/logstash -f /mnt/app/logstash/conf/test.conf
Settings: Default pipeline workers: 2
Pipeline main started

[root@smallasa test]# echo ‘101.201.57.207 - [11/May/2017:10:09:12 +0800] “POST /gateway-inbound/rest/inv.xml HTTP/1.1” 200 566 “-“ - 0.013 0.011’ | tee -a inbound.log

[root@smallasa conf]# /mnt/app/logstash/bin/logstash -f /mnt/app/logstash/conf/test.conf
Settings: Default pipeline workers: 2
Pipeline main started
{
“message” => “101.201.57.207 - [11/May/2017:10:09:12 +0800] \”POST /gateway-inbound/rest/inv.xml HTTP/1.1\” 200 566 \”-\” - 0.013 0.011”,
“@version” => “1”,
“@timestamp” => “2017-05-14T02:59:53.713Z”,
“path” => “/mnt/log/nginx/test/inbound.log”,
“host” => “smallasa”,
“type” => “nginx_log”

}

[root@smallasa conf]# cat test.conf
input{
file {
path => [“/mnt/log/nginx/test/inbound.log”]
exclude => [“kibana.log”,”toolstmp.log”,”pennswitch*.log”]
type => “nginx_log”
discover_interval => 15
sincedb_path => “/mnt/data/logstash”
sincedb_write_interval => 15
stat_interval => 1
start_position => “beginning”
}
}
filter{
if [type] == “nginx_log” {
grok {
match => [“message”, “%{COMBINEDAPACHELOG}”]
}
}
}
output{
stdout{
codec => rubydebug
}
}
[root@smallasa conf]# /mnt/app/logstash/bin/logstash -f /mnt/app/logstash/conf/test.conf –configtest
Configuration OK

[root@smallasa conf]# /mnt/app/logstash/bin/logstash -f /mnt/app/logstash/conf/test.conf
Settings: Default pipeline workers: 2
Pipeline main started

[root@smallasa test]# echo ‘101.201.57.207 - [11/May/2017:10:09:12 +0800] “POST /gateway-inbound/rest/inv.xml HTTP/1.1” 200 566 “-“ - 0.013 0.011’ | tee -a inbound.log

[root@smallasa conf]# /mnt/app/logstash/bin/logstash -f /mnt/app/logstash/conf/test.conf
Settings: Default pipeline workers: 2
Pipeline main started
{
“message” => “101.201.57.207 - [11/May/2017:10:09:12 +0800] \”POST /gateway-inbound/rest/inv.xml HTTP/1.1\” 200 566 \”-\” - 0.013 0.011”,
“@version” => “1”,
“@timestamp” => “2017-05-14T03:07:45.105Z”,
“path” => “/mnt/log/nginx/test/inbound.log”,
“host” => “smallasa”,
“type” => “nginx_log”,
“tags” => [
[0] “_grokparsefailure”
]
}