hadoop►logstash►logstash config file

logstash conf file

logstash 通过log4j收集日志,写入ES

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
// log4j-4501
input{
  log4j{
    mode => "server"
    host => "10.0.3.41"
    port => 4501
  }
}
filter{
  urldecode {
    all_fields => true
  }
  mutate {
    remove_field => [ "tags","timestamp" ]
  }
}
output{
  elasticsearch{
    action => "index"
    index => "xxx-%{[application]}-%{+YYYY.MM}"
    hosts => ["10.0.3.40:9200","10.0.3.41:9200","10.0.3.42:9200"]
  }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
//log4j-4502
input{
  log4j{
    mode => "server"
    host => "10.0.3.42"
    port => 4502
  }
}
filter{
  json {
    source => "message"
  }
  urldecode {
    all_fields => true
  }
  mutate {
    remove_field => [ "tags" ]
  }
}
output{
  elasticsearch{
    action => "index"
    index => "xxx-%{[application]}-%{+yyyy.MM}"
    document_type => "%{[key]}"
    hosts => ["10.0.3.40:9200","10.0.3.41:9200","10.0.3.42:9200"]
  }
}