2. 技术
    3. ELK日志分析架构部署

    ELK日志分析架构部署

    技术2022-07-11  79

    准备环境

    关闭防火墙和SELinux 10.8.156.134 es1 10.8.156.135 es2 10.8.156.136 es3

    ELK下载包官网地址链接

    #三台机器都要配置jdk [xiaobai@es1] tar xzf jdk-8u211-linux-x64.tar.gz -C /usr/local/ [xiaobai@es1] cd /usr/local/ [xiaobai@es1 local] mv jdk1.8.0_191/ java [xiaobai@es1 local] echo ' JAVA_HOME=/usr/local/java PATH=$JAVA_HOME/bin:$PATH export JAVA_HOME PATH ' >>/etc/profile [xiaobai@es1 local] source /etc/profile [xiaobai@es1 local] java -version java version "1.8.0_211" Java(TM) SE Runtime Environment (build 1.8.0_211-b12) Java HotSpot(TM) 64-Bit Server VM (build 25.211-b12, mixed mode)

    Elasticsearch部署

    #以下只在es1机器上操作 [xiaobai@es1] useradd elsearch [xiaobai@es1] echo "123456" | passwd --stdin "elsearch" [xiaobai@es1] tar xzf elasticsearch-7.8.0-linux-x86_64.tar.gz -C /usr/local/ [xiaobai@es1] cd /usr/local/elasticsearch-7.8.0/config/ [xiaobai@es1 config] ls elasticsearch.yml jvm.options.d role_mapping.yml users jvm.options log4j2.properties roles.yml users_roles [xiaobai@es1 config] cp elasticsearch.yml elasticsearch.yml.bak [xiaobai@es1 config] vim elasticsearch.yml cluster.name: elk node.name: elk01 node.master: true node.data: true path.data: /data/elasticsearch/data path.logs: /data/elasticsearch/logs bootstrap.memory_lock: false bootstrap.system_call_filter: false network.host: 0.0.0.0 http.port: 9200 #discovery.zen.ping.unicast.hosts: ["10.8.156.134","10.8.156.135","10.8.156.136"] ##discovery.zen.minimum_master_nodes: 2 ##discovery.zen.ping_timeout: 150s ##discovery.zen.fd.ping_retries: 10 ##client.transport.ping_timeout: 60s http.cors.enabled: true http.cors.allow-origin: "*"

    cluster.name 集群名称，各节点配成相同的集群名称。 node.name 节点名称，各节点配置不同。 node.master 指示某个节点是否符合成为主节点的条件。 node.data 指示节点是否为数据节点。数据节点包含并管理索引的一部分。 path.data 数据存储目录。 path.logs 日志存储目录。 bootstrap.memory_lock 内存锁定，是否禁用交换。 bootstrap.system_call_filter 系统调用过滤器。 network.host 绑定节点IP。 http.port 端口。 discovery.zen.ping.unicast.hosts 提供其他 Elasticsearch 服务节点的单点广播发现功能。 discovery.zen.minimum_master_nodes 集群中可工作的具有Master节点资格的最小数量，官方的推荐值是(N/2)+1，其中N是具有master资格的节点的数量。 discovery.zen.ping_timeout 节点在发现过程中的等待时间。 discovery.zen.fd.ping_retries 节点发现重试次数。 http.cors.enabled 是否允许跨源 REST 请求，用于允许head插件访问ES。 http.cors.allow-origin 允许的源地址。

    #设置jvm堆大小 [xiaobai@es1 config] vim jvm.options -Xms1g #将-Xms1g ----修改成 -Xms2g -Xms1g #将-Xmx1g ----修改成 -Xms2g #推荐设置为4G，也可以选用下面的配置 [xiaobai@es1 config] sed -i 's/-Xms1g/-Xms4g/' /usr/local/elasticsearch-7.8.0/config/jvm.options [xiaobai@es1 config] sed -i 's/-Xms1g/-Xms4g/' /usr/local/elasticsearch-7.8.0/config/jvm.options sed -i 's/-Xmx1g/-Xmx4g/' /usr/local/elasticsearch-6.5.4/config/jvm.options #以上两种选一种 #确保堆内存最小值（Xms）与最大值（Xmx）的大小相同，防止程序在运行时改变堆内存大小。堆内存大小不要超过系统内存的50% #创建es数据及日志存储目录 [xiaobai@es1 config] cd [xiaobai@es1] mkdir -p /data/elasticsearch/data [xiaobai@es1] mkdir -p /data/elasticsearch/logs [xiaobai@es1] chown -R elsearch:elsearch /data/elasticsearch [xiaobai@es1] chown -R elsearch:elsearch /usr/local/elasticsearch-7.8.0/ #增加最大文件打开数 [xiaobai@es1] echo "* - nofile 65536" >> /etc/security/limits.conf #增加最大进程数 [xiaobai@es1] vim /etc/security/limits.conf #在文件最后一行加入 * soft nofile 65536 * hard nofile 131072 * soft nproc 2048 * hard nproc 4096 #soft xxx : 代表警告的设定，可以超过这个设定值，但是超过后会有警告。 #hard xxx : 代表严格的设定，不允许超过这个设定的值。 #nofile : 是每个进程可以打开的文件数的限制 #nproc : 是操作系统级别对每个用户创建的进程数的限制 #增加最大内存映射数 [xiaobai@es1] vim /etc/sysctl.conf #在最后一行加入 vm.max_map_count=262144 vm.swappiness=0 [xiaobai@es1] sysctl -p [xiaobai@es1] sysctl -w vm.max_map_count=262144 #如果启动报错 memory locking requested for elasticsearch process but memory is not locked.elasticsearch.yml.bootstrap.memory_lock : false /etc/sysctl.conf vm.swappiness=0 #错误 max file descriptors [4096] for elasticsearch process is too low, increase to at least [65536] [xiaobai@es1] vim /etc/security/limits.conf #在最后一行添加 * hard nofile 65536 * hard nofile 65536 #启动还会遇到另外一个问题 max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144] #这个比较简单，也不需要重启，直接执行 [xiaobai@es1] sysctl -w vm.max_map_count=262144 #意思是：elasticsearch用户拥有的内存权限太小了，至少需要262114。 [xiaobai@es1] su - elsearch [elsearch@es1] cd /usr/local/elasticsearch-7.8.0/ [elsearch@es1 elasticsearch-7.8.0] nohup ./bin/elasticsearch & [elsearch@es1 elasticsearch-7.8.0] tail -f nohup.out

    安装监控插件

    node插件安装链接，下面有wget下载地址

    #在es2机器上操作 #安装node [xiaobai@es2] wget https://npm.taobao.org/mirrors/node/latest-v4.x/node-v4.4.7-linux-x64.tar.gz [xiaobai@es2] ls node-v4.4.7-linux-x64.tar.gz [xiaobai@es2] tar xzf node-v4.4.7-linux-x64.tar.gz -C /usr/local/ [xiaobai@es2] vim /etc/profile NODE_HOME=/usr/local/node-v4.4.7-linux-x64 PATH=$NODE_HOME/bin:$PATH export NODE_HOME PATH [xiaobai@es2] source /etc/profile [xiaobai@es2] node --version v4.4.7 #在es2机器上操作 #下载head插件 [xiaobai@es2] wget https://github.com/mobz/elasticsearch-head/archive/master.zip [xiaobai@es2] cp master.zip /usr/local/ [xiaobai@es2] yum -y install unzip [xiaobai@es2] cd /usr/local/ [xiaobai@es2 local] unzip master.zip #在es2机器上操作 #安装grunt [xiaobai@es2 local] cd elasticsearch-head-master/ [xiaobai@es2 elasticsearch-head-master] npm install -g grunt-cli [xiaobai@es2 elasticsearch-head-master] grunt --version grunt-cli v1.3.2 #在es2机器上操作 #修改head源码 [xiaobai@es2 elasticsearch-head-master] vim /usr/local/elasticsearch-head-master/Gruntfile.js

    [xiaobai@es2 elasticsearch-head-master] vim /usr/local/elasticsearch-head-master/_site/app.js

    #下载head必要的文件 [xiaobai@es2] wget https://github.com/Medium/phantomjs/releases/download/v2.1.1/phantomjs-2.1.1-linux-x86_64.tar.bz2 [xiaobai@es2] yum -y install bzip2 [xiaobai@es2] tar xjf phantomjs-2.1.1-linux-x86_64.tar.bz2 -C /tmp/

    启动head

    #在es2机器上操作 [xiaobai@es2] cd /usr/local/elasticsearch-head-master/ [xiaobai@es2 elasticsearch-head-master] npm config set registry https://registry.npm.taobao.org [xiaobai@es2 elasticsearch-head-master] npm install phantomjs-prebuilt@2.1.16 --ignore-scripts [xiaobai@es2 elasticsearch-head-master] nohup grunt server & [xiaobai@es2 elasticsearch-head-master] tail -f nohup.out

    Kibana部署

    #开头文章有kibana的下载包地址 #在es2机器上操作 [xiaobai@es2] tar xzf kibana-7.8.0-linux-x86_64.tar.gz -C /usr/local/ [xiaobai@es2] cd /usr/local/kibana-7.8.0-linux-x86_64/config/ [xiaobai@es2 config] vim kibana.yml

    server.port kibana 服务端口，默认5601 server.host kibana 主机IP地址，默认localhost elasticsearch.hosts 用来做查询的ES节点的hosts，默认http://localhost:9200 kibana.index kibana在Elasticsearch中使用索引来存储保存searches，visualizations和dashboards。默认.kibana

    启动kibana

    #启动kibana [xiaobai@es2 config] cd ../bin [xiaobai@es2 bin] nohup ./kibana --allow-root & #--allow-root，允许root用户启动 #[2] 8884 #[root@es2 bin]# nohup: ignoring input and appending output to ‘nohup.out’ [xiaobai@es2 bin] netstat -tnlp Active Internet connections (only servers) Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 7081/sshd tcp 0 0 127.0.0.1:25 0.0.0.0:* LISTEN 7204/master tcp 0 0 10.8.156.135:5601 0.0.0.0:* LISTEN 8824/./bin/../node/ tcp6 0 0 :::9100 :::* LISTEN 8042/grunt tcp6 0 0 :::22 :::* LISTEN 7081/sshd tcp6 0 0 ::1:25 :::* LISTEN 7204/master

    #nginx反向代理 [xiaobai@es2 bin] cd [xiaobai@es2] rpm -ivh http://nginx.org/packages/centos/7/noarch/RPMS/nginx-release-centos-7-0.el7.ngx.noarch.rpm [xiaobai@es2] yum -y install nginx [xiaobai@es2] cd /etc/nginx/conf.d/ [xiaobai@es2 conf.d] cp default.conf nginx.conf.bak [xiaobai@es2 conf.d] mv default.conf nginx.conf [xiaobai@es2 conf.d] vim nginx.conf server { listen 80; server_name 10.8.156.135; #charset koi8-r; # access_log /var/log/nginx/host.access.log main; # access_log off; location / { proxy_pass http://10.8.156.135:5601; proxy_set_header Host $host:5601; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Via "nginx"; } location /status { stub_status on; #开启网站监控状态 access_log /var/log/nginx/kibana_status.log; #监控日志 auth_basic "NginxStatus"; } location /head/ { proxy_pass http://10.8.156.135:9100; proxy_set_header Host $host:9100; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Via "nginx"; } } [xiaobai@es2 conf.d] cd .. [xiaobai@es2 nginx] vim /etc/nginx/nginx.conf #将原来的log_format注释掉，添加json格式的配置信息 log_format json '{"@timestamp":"$time_iso8601",' '"@version":"1",' '"client":"$remote_addr",' '"url":"$uri",' '"status":"$status",' '"domain":"$host",' '"host":"$server_addr",' '"size":$body_bytes_sent,' '"responsetime":$request_time,' '"referer": "$http_referer",' '"ua": "$http_user_agent"' '}'; access_log /var/log/nginx/access_json.log json;

    #启动nginx [xiaobai@es2 nginx] systemctl start nginx [xiaobai@es2 nginx] systemctl enable nginx

    Logstash部署

    #文章开头有logstash下载包的链接 #在es3机器上操作 [xiaobai@es3] tar xzf logstash-7.8.0.tar.gz -C /usr/local/ #安装nginx,用于模拟生产环境中的服务，收集nginx日志 [xiaobai@es3] rpm -ivh http://nginx.org/packages/centos/7/noarch/RPMS/nginx-release-centos-7-0.el7.ngx.noarch.rpm [xiaobai@es3] yum -y install nginx [xiaobai@es3] vim /etc/nginx/nginx.conf #同上，将原来的日志格式注释掉定义成json格式 log_format json '{"@timestamp":"$time_iso8601",' '"@version":"1",' '"client":"$remote_addr",' '"url":"$uri",' '"status":"$status",' '"domain":"$host",' '"host":"$server_addr",' '"size":$body_bytes_sent,' '"responsetime":$request_time,' '"referer": "$http_referer",' '"ua": "$http_user_agent"' '}'; access_log /var/log/nginx/access_json.log json; [xiaobai@es3] systemctl start nginx [xiaobai@es3] systemctl enable nginx

    [xiaobai@es3] mkdir -p /usr/local/logstash-7.8.0/etc/conf.d [xiaobai@es3] cd /usr/local/logstash-7.8.0/etc/conf.d/ [xiaobai@es3 conf.d] vim input.conf input { #让logstash可以读取特定的事件源。 file { #从文件读取 path => ["/var/log/nginx/access_json.log"] #要输入的文件路径 # code => "json" #定义编码，用什么格式输入和输出，由于日志就是json格式，这里不用再写 type => "shopweb" #定义一个类型，通用选项. 用于激活过滤器 } } [xiaobai@es3 conf.d] vim output.conf output { #输出插件，将事件发送到特定目标 elasticsearch { #输出到es机器 hosts => ["10.8.156.134:9200"] #指定es服务的ip加端口 index => ["%{type}-%{+YYYY.MM.dd}"] #引用input中的type名称，定义输出的格式 } }

    启动logstash

    [xiaobai@es3 conf.d] cd ../.. [xiaobai@es3 logstash-7.8.0] nohup bin/logstash -f etc/conf.d/ --config.reload.automatic & [xiaobai@es3 logstash-7.8.0] tail -f nohup.out

    head上查看

    kibana上查看

    文章总览分类目录 @小白

    Processed: 0.015, SQL: 9