一、下载地址
官网：https://www.elastic.co/cn/downloads/logstash
百度云盘：
二、安装
tar zxvf logstash-6.2.1.tar.gz
mv logstash-6.2.1 logstash
配置文件（配置文件放哪个目录都可以，在启动Logstash时可以指定配置文件位置）
input{
kafka {
bootstrap_servers => "10.10.6.225:9092" #kafka服务器地址，不是zookeeper
client_id => "test"
auto_offset_reset => "latest"
consumer_threads => 5
decorate_events => true
topics => ["test"] #控制kafka哪个topic，可以多个用逗号分割
codec => "json"
}
}
filter{
json {
source => "message"
remove_field=>["message","beat","@version"] #删除没用的属性
add_field =>["customize","自定义字段"] #增加属性
}
}
output {
elasticsearch {
hosts => "localhost:9200"
index => "logstash-nginxacclog-%{+YYYY.MM.dd}" #放到elasticsearch哪个index中
}
}
启动（到bin目录下）
./logstash -f k_es.conf
后台运行
nohup /usr/local/logstash/bin/logstash -f /usr/local/logstash/bin/k_es.conf -w 8 -b 1000 > /dev/null 2>&1 &
配置文件备份
input{
kafka {
bootstrap_servers => "10.100.2.210:9092"
client_id => "nginxaccesslog"
auto_offset_reset => "latest"
consumer_threads => 5
decorate_events => true
topics => ["nginx_access_log"]
codec => "json"
type => "nginx_log"
}
kafka {
bootstrap_servers => "10.100.2.210:9092"
client_id => "database"
auto_offset_reset => "latest"
consumer_threads => 5
decorate_events => true
topics => ["t_resource_info","t_base_person","t_base_organization","t_base_student","t_base_parent","t_base_class"]
codec => "json"
type => "dsideal_db"
}
kafka {
bootstrap_servers => "10.100.2.210:9092"
client_id => "devops_real"
auto_offset_reset => "latest"
consumer_threads => 5
decorate_events => true
topics => ["devopsrealinfo"]
codec => "json"
type => "devopsrealinfo"
}
kafka {
bootstrap_servers => "10.100.2.210:9092"
client_id => "devops_base"
auto_offset_reset => "latest"
consumer_threads => 5
decorate_events => true
topics => ["devopsbaseinfo"]
codec => "json"
type => "devopsbaseinfo"
}
}
filter{
mutate {
gsub => ["message", "\\x22", '"']
}
json {
source => "message"
remove_field=>["message","beat","@version"]
}
if [type] == "nginx_log" {
geoip {
source => "ip"
target => "geoip"
database => "/usr/local/GeoLite2-City.mmdb"
}
}
}
output {
if [type] == "nginx_log" {
elasticsearch {
hosts => "10.100.2.210:9200"
index => "nginx-access-log"
}
}
if [type] == "dsideal_db" {
elasticsearch {
hosts => "10.100.2.210:9200"
index => "%{table_name}"
document_id => "%{id}"
}
}
if [type] == "devopsbaseinfo" {
elasticsearch {
hosts => "localhost:9200"
index => "devopsbaseinfo"
document_id => "%{id}"
}
}
if [type] == "devopsrealinfo" {
elasticsearch {
hosts => "localhost:9200"
index => "devopsrealinfo"
}
}
}
