Shipper->Broker->Indexer->ES
1.input
input { stdin {} }
output {
stdout { codec=> rubydebug }
}
file
{
codec => multiline {
pattern =>
"^\s"
what =>
"previous"
}
path => [
"xx"
,
"xx"
]
exclude =>
"1.log"
add_field => [
"log_ip"
,
"xx"
]
tags =>
"tag1"
delimiter =>
"\n"
discover_interval => 15
stat_interval => 1
start_position => beginning
sincedb_path =>
"E:/software/logstash-1.5.4/logstash-1.5.4/test.txt"
sincedb_write_interval => 15
}
2.filter
filter {
multiline {
pattern =>
"^[^\d]"
what =>
"previous"
}
filter {
multiline {
type
=>
"type"
pattern =>
"pattern, a regexp"
negate => boolean
what =>
"previous"
or
"next"
}
}
filter {
grep
{
match => [
"@message"
,
"PHP Fatal error"
]
drop =>
false
add_tag => [fatal_error]
}
grep
{
tags => [fatal_error]
match => [
"@message"
,
".*(xbox\.com|xbox\.mib\.com\.cn|supports\.game\.mib\.com\.cn)"
]
drop =>
false
add_tag => [xboxerror]
}
}
filter {
if
[message] !~
"5.3.3|down"
{
ruby {
code =>
"event.cancel"
}
}
}
filter {
grok {
match => {
"message"
=>
"%{COMBINEDAPACHELOG}"
}
}
}
filter {
multiline {
pattern =>
"^[^[]"
negate =>
true
what =>
"previous"
}
}
filter {
if
[path] =~
"error"
{
mutate { replace => {
"type"
=>
"apache_error"
} }
grok {
match => {
"message"
=>
"%{COMBINEDAPACHELOG}"
}
}
}
date
{
match => [
"timestamp"
,
"dd/MMM/yyyy:HH:mm:ss Z"
]
}
}
filter {
if
[path] =~
"access"
{
mutate { replace => {
type
=>
"apache_access"
} }
grok {
match => {
"message"
=>
"%{COMBINEDAPACHELOG}"
}
}
date
{
match => [
"timestamp"
,
"dd/MMM/yyyy:HH:mm:ss Z"
]
}
}
else
if
[path] =~
"error"
{
mutate { replace => {
type
=>
"apache_error"
} }
}
else
{
mutate { replace => {
type
=>
"random_logs"
} }
}
}
3.output
发邮件
output {
email {
match => [
"@message"
,
"aaaaa"
]
to =>
"storyskya@gmail.com"
from =>
"monitor@mib.com.cn"
options => [
"smtpIporHost"
,
"smtp.mibnet.com"
,
"port"
,
"25"
,
"userName"
,
"monitor@mib.com.cn"
,
"starttls"
,
"true"
,
"password"
,
"opmonitor"
,
"authenticationType"
,
"login"
]
subject =>
"123"
body =>
'123'
via => smtp
}
}
output {
if
[
type
] ==
"syslog"
{
elasticsearch {
hosts =>
"172.16.0.102:9200"
index =>
"syslog-%{+YYYY.MM.dd}"
}
}
if
[
type
] ==
"nginx"
{
elasticsearch {
hosts =>
"172.16.0.102:9200"
index =>
"nglog-%{+YYYY.MM.dd}"
}
}
if
[message] =~
/paramiko
|simplejson/ {
email {
to =>
"12222222@wo.cn"
from =>
"good_zabbix@163.com"
contenttype =>
"text/plain; charset=UTF-8"
address =>
"smtp.163.com"
username =>
"test@163.com"
password =>
"12344"
subject =>
"服务器%{host}日志异常"
body =>
"%{@timestamp} %{type}: %{message}"
}
}
}
output {
stdout { codec => rubydebug }
redis {
host =>
'192.168.1.104'
data_type =>
'list'
key =>
'logstash:redis'
}
}
output {
elasticsearch { host => localhost }
stdout { codec => rubydebug }
}
替换
mutate {
type
=>
"phplog"
gsub => [
"@message"
,
"'"
,
"\""
]
}
调试
curl '
logstash -e
'input{stdin{}}output{stdout{codec=>rubydebug}}'
