c4rt1y

ELK环境安装及使用

0x01 环境安装

#两台服务器
master 10.10.10.20
node01 10.10.10.30

#在master和node01上
#因为elasticsearch基于java环境,所以首先安装java
yum install -y java
java -version

#准备elasticsearch的GPG-KEY
rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch

#准备elasticsearch源
cat > /etc/yum.repos.d/elasticsearch.repo <<OFF
[elasticsearch-2.x]
name=Elasticsearch repository for 2.x packages
baseurl=http://packages.elastic.co/elasticsearch/2.x/centos
gpgcheck=1
gpgkey=http://packages.elastic.co/GPG-KEY-elasticsearch
enabled=1
OFF

#安装elasticsearch
yum install -y elasticsearch

mkdir -p  /data/
chown -R elasticsearch.elasticsearch /data/

#配置elasticsearch
cat > /etc/elasticsearch/elasticsearch.yml <<OFF
cluster.name: test        
node.name: node
path.data: /data/
path.logs: /var/log/elasticsearch/
bootstrap.mlockall: true
network.host: 0.0.0.0
http.port: 9200
discovery.zen.ping.multicast.enabled: false
discovery.zen.ping.unicast.hosts: ["10.10.10.20", "10.10.10.30"]
OFF

#elasticsearch文件配置如下
grep '^[a-Z]' /etc/elasticsearch/elasticsearch.yml 	【将里面内容情况,配置下面内容】
cluster.name: test 	# 组名(同一个组,组名必须一致)
node.name: master 	 # 节点名称,建议和主机名一致
path.data: /data/es-data 	# 数据存放的路径
path.logs: /var/log/elasticsearch/ 	# 日志存放的路径
bootstrap.mlockall: true 	# 锁住内存,不被使用到交换分区去
network.host: 0.0.0.0 		# 网络设置
http.port: 9200		# 端口
discovery.zen.ping.multicast.enabled: false	# 主播  [自动搜索存在elasticsearch主机,不过在centos7中不适用]
discovery.zen.ping.unicast.hosts: ["10.10.10.20", "10.10.10.30"] # 单播  [自定义有哪些服务器作为elasticsearch设备]


#node01上的差别是name
grep '^[a-Z]' /etc/elasticsearch/elasticsearch.yml
cluster.name: test        
node.name: node01 	#name不一样
path.data: /data/
path.logs: /var/log/elasticsearch/
bootstrap.mlockall: true
network.host: 0.0.0.0
http.port: 9200
discovery.zen.ping.multicast.enabled: false
discovery.zen.ping.unicast.hosts: ["10.10.10.20", "10.10.10.30"]

#设置启动和启动elasticsearch
systemctl start elasticsearch
systemctl status elasticsearch

#添加logstash yum仓库
cat > /etc/yum.repos.d/logstash.repo <<OFF
[logstash-2.1]
name=Logstash repository for 2.1.x packages
baseurl=http://packages.elastic.co/logstash/2.1/centos
gpgcheck=1
gpgkey=http://packages.elastic.co/GPG-KEY-elasticsearch
enabled=1
OFF

#安装logstash
yum install -y logstash

logstash启动
systemctl restart elasticsearch
systemctl status elasticsearch

#安装head插件
#方法一
/usr/share/elasticsearch/bin/plugin install mobz/elasticsearch-head
#方法二(github下载,自己放到elasticsearch插件目录)
https://github.com/mobz/elasticsearch-head
mkdir /usr/share/elasticsearch/plugins/head
cp -r /usr/local/src/elasticsearch-head-master/* elasticsearch-head/*
chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/plugins
#安装kopf插件
/usr/share/elasticsearch/bin/plugin install lmenezes/elasticsearch-kopf

#安装完插件,重启elasticsearch
systemctl restart elasticsearch

#浏览器访问 
http://10.10.10.20:9200/_plugin/head/
http://10.10.10.20:9200/_plugin/kopf/


#Kibana安装配置
#kibana的安装:
wget https://download.elastic.co/kibana/kibana/kibana-4.3.1-linux-x64.tar.gz
tar zxf kibana-4.3.1-linux-x64.tar.gz
mv kibana-4.3.1-linux-x64 /usr/local/
ln -s /usr/local/kibana-4.3.1-linux-x64/ /usr/local/kibana

#修改配置文件
cp /usr/local/kibana/config/kibana.yml /usr/local/kibana/config/kibana.yml.bak
vi /usr/local/kibana/config/kibana.yml
server.port: 5601
server.host: "0.0.0.0"
elasticsearch.url: "http://10.10.10.20:9200"
kibana.index: ".kibana"

#因为他一直运行在前台,要么选择开一个窗口,要么选择使用screen。安装并使用screen启动kibana:
yum -y install screen
screen                         
/usr/local/kibana/bin/kibana

#然后按ctrl+a+d组合键,这样在上面另启的screen屏里启动的kibana服务就一直运行在前台了....,使用screen -ls测试是否运行
screen -ls
There is a screen on:
15041.pts-0.elk-node1 (Detached)

#访问 10.10.10.20:5601,加入参数,会有图像和内容


#对于两台服务器的配置不同
elasticsearch是通过主播或者单播的形式,根据cluster.name是否相同来找同步服务器,其中node.name应该可以说是配置不一样,其他可以保持一样。

0x02 ELK使用

#通过curl,查询elasticsearch的日志
curl -i -XGET 'http://10.10.10.20:9200/_count?pretty' -d '{"query":{"match_all":{}}}'

#logstash使用
#输入内容,输出内容
/opt/logstash/bin/logstash -e 'input { stdin{} } output { stdout{} }'

#输入内容,详细输出
/opt/logstash/bin/logstash -e 'input { stdin{} } output { stdout{ codec => rubydebug} }'

#输入内容,输出内容,并保存到elasticsearch里面
/opt/logstash/bin/logstash -e 'input { stdin{} } output { elasticsearch { hosts => ["10.10.10.20:9200"]} }'

#输入内容,详细输出,并保存到elasticsearch里面
/opt/logstash/bin/logstash -e 'input { stdin{} } output { elasticsearch { hosts => ["10.10.10.20:9200"]} stdout{ codec => rubydebug}}'

#监控脚本01-收集系统日志[收集系统日志,并将内容写入elasticsearch]
vi /etc/logstash/conf.d/sysfile-logstash.conf
input {
    file {
      path => "/var/log/messages"
      type => "system"
      start_position => "beginning"
    }
    syslog {
        type => "system-syslog"
        host => "192.168.1.160"
        port => "514"
    }
}
 
output {
    if [type] == "system"{
        elasticsearch {
           hosts => ["192.168.1.160:9200"]
           index => "system-%{+YYYY.MM.dd}"
        }
    }
 
    if [type] == "system-syslog"{
        elasticsearch {
           hosts => ["192.168.1.160:9200"]
           index => "system-syslog-%{+YYYY.MM.dd}"
        }
    }
}

#检测语法是否正确
/opt/logstash/bin/logstash -f /etc/logstash/conf.d/sysfile-logstash.conf --configtest

#运行程序
/opt/logstash/bin/logstash -f /etc/logstash/conf.d/sysfile-logstash.conf


#监控脚本01-收集nginx的访问日志[收集系统日志,并将内容写入elasticsearch]
修改nginx的配置文件,分别在nginx.conf的http和server配置区域添加下面内容:
vi /etc/nginx/nginx.conf
#http 标签中
          log_format json '{"@timestamp":"$time_iso8601",'
                           '"@version":"1",'
                           '"client":"$remote_addr",'
                           '"url":"$uri",'
                           '"status":"$status",'
                           '"domain":"$host",'
                           '"host":"$server_addr",'
                           '"size":$body_bytes_sent,'
                           '"responsetime":$request_time,'
                           '"referer": "$http_referer",'
                           '"ua": "$http_user_agent"'
'}';
#server标签中
            access_log /var/log/nginx/access_json.log json;

#收集nginx的访问日志[收集系统日志
vi /etc/logstash/conf.d/nginx-logstash.conf
input {
	    file {
	       path => "/var/log/nginx/access_json.log"
	       codec => json
	       start_position => "beginning"
	       type => "nginx-log"
	    }
	}
	 
	output {
        elasticsearch {
           hosts => ["192.168.1.160:9200"]
           index => "nignx-log-%{+YYYY.MM.dd}"
        }
	}



#收集IIS的访问日志
vi /etc/logstash/conf.d/iis-logstash.conf
input {
  file {
    type => "iis_log_1"
    path => ["C:/inetpub/logs/LogFiles/W3SVC1/*.log"]
    start_position => "beginning"
  }
}
filter {
  if [type] == "iis_log_1" {
  #ignore log comments
  if [message] =~ "^#" {
    drop {}
  }
  grok {
    # check that fields match your IIS log settings
    match => ["message", "%{TIMESTAMP_ISO8601:log_timestamp} %{IPORHOST:site} %{WORD:method} %{URIPATH:page} %{NOTSPACE:querystring} %{NUMBER:port} %{NOTSPACE:username} %{IPORHOST:clienthost} %{NOTSPACE:useragent} %{NUMBER:response} %{NUMBER:subresponse} %{NUMBER:scstatus} %{NUMBER:time_taken}"]
  }
    date {
    match => [ "log_timestamp", "YYYY-MM-dd HH:mm:ss" ]
      timezone => "Etc/UTC"
  }    
  useragent {
    source=> "useragent"
    prefix=> "browser"
  }
  mutate {
    remove_field => [ "log_timestamp"]
  }
  }
}
output {
  if [type] == "iis_log_1" {
  logservice {
        codec => "json"
        endpoint => "***"
        project => "***"
        logstore => "***"
        topic => ""
        source => ""
        access_key_id => "***"
        access_key_secret => "***"
        max_send_retry => 10
    }
    }
}



#由于elasticsearch的日志收集时可能会出现卡顿,所以我们可以使用redis进行缓存
#通过redis写入
vi /etc/logstash/conf.d/all_to_redis.conf
input {
    file {
      path => "/var/log/messages"
      type => "system"
      start_position => "beginning"
    }
    file {
       path => "/var/log/nginx/access_json.log"
       codec => json
       start_position => "beginning"
       type => "nginx-log"
    }
    syslog {
        type => "system-syslog"
        host => "192.168.1.160"
        port => "514"
    }
  
}
output {
   if [type] == "system"{
     redis {
        host => "192.168.1.160"
        port => "6379"
        db => "6"
        data_type => "list"
        key => "system"
     }
   }
    if [type] == "nginx-log"{   
       redis {
          host => "192.168.1.160"
          port => "6379"
          db => "6"
          data_type => "list"
          key => "nginx-log"
       }
    }
    if [type] == "system-syslog"{
       redis {
          host => "192.168.1.160"
          port => "6379"
          db => "6"
          data_type => "list"
          key => "system-syslog"
       }   
     }
}

#通过读取redis获取
vi /etc/logstash/conf.d/redis_to_elasticsearch.conf
input {
     redis {
        type => "system"
        host => "192.168.1.160"
        port => "6379"
        db => "6"
        data_type => "list"
        key => "system"
     }
     redis {
        type => "nginx-log"
        host => "192.168.1.160"
        port => "6379"
        db => "6"
        data_type => "list"
        key => "nginx-log"
     }
     redis {
        type => "system-syslog"
        host => "192.168.1.160"
        port => "6379"
        db => "6"
        data_type => "list"
        key => "system-syslog"
     } 
}
output {
 
    if [type] == "system"{
        elasticsearch {
           hosts => ["192.168.1.160:9200"]
           index => "system-%{+YYYY.MM.dd}"
        }
    }
    if [type] == "nginx-log"{
        elasticsearch {
           hosts => ["192.168.1.160:9200"]
           index => "nignx-log-%{+YYYY.MM.dd}"
        }
    }
    if [type] == "system-syslog"{
        elasticsearch {
           hosts => ["192.168.1.160:9200"]
           index => "system-syslog-%{+YYYY.MM.dd}"
        }
    }
}

0x03 ELK优化

算起来,应该是jvm+flush_size优化,关键还是业务主机的真实情况问题。

核心:worker * batch_size / flush_size = ES bulk index api 调用次数

1.根据CPU核数调整合适的worker数量,观察系统负载。
2.根据内存堆大小,调整batch_size,调试JVM,观察GC,线程是否稳定。
3.调整flush_size,这个值默认500,我在生产环境使用的1500,这个值需要你逐步增大,观察性能,增大到一定程度时,性能会下降,峰值还是需要观看真实情况。

0x04 参考文献

https://doc.yonyoucloud.com/doc/logstash-best-practice-cn/index.html
https://kibana.logstash.es/content/
https://www.gitbook.com/book/chenryn/elk-stack-guide-cn/details
http://www.cnblogs.com/kevingrace/p/5919021.html#3770932
https://help.aliyun.com/document_detail/28993.html?spm=5176.doc43757.6.615.NCxz15
http://blog.csdn.net/u014297722/article/category/6861245
GoTop