Elastic Stack 5.1 部署

迈不过友情╰ 2022-06-12 14:54 174阅读 0赞

#### Elastic Stack安装步骤 ####

*  1、Elasticsearch  
    X-Pack for Elasticsearch
 *  2、Kibana  
    X-Pack for Kibana
 *  3、Logstash
 *  4、Beats
 *  5、Elasticsearch Hadoop
 *  6、redis

#### 参考文档 ####

[https://www.elastic.co/guide/en/elastic-stack/current/installing-elastic-stack.html][https_www.elastic.co_guide_en_elastic-stack_current_installing-elastic-stack.html]

#### 1、安装Elasticsearch & X-pack（yum安装） ####

X-Pack是一个Elastic Stack的扩展，将安全，警报，监视，报告和图形功能包含在一个易于安装的软件包中。x-Pack安装之后有一个超级用户elastic ，其默认的密码是changeme，拥有对所有索引和数据的控制权，可以使用该用户创建和修改其他用户。通过kibana的web界面进行用户和用户组的管理，也可以使用shell 终端进行管理。

[root@tzgsqvelk01 tzg]# rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch
    
    [root@tzgsqvelk01 tzg]# cat > /etc/yum.repos.d/elasticsearch.repo <<-EOF
    [elasticsearch-5.x]
    name=Elasticsearch repository for 5.x packages
    baseurl=https://artifacts.elastic.co/packages/5.x/yum
    gpgcheck=1
    gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
    enabled=1
    autorefresh=1
    type=rpm-md
    EOF
    
    [root@tzgsqvelk01 tzg]# yum install elasticsearch -y
    
    [root@tzgsqvelk01 tzg]# /bin/systemctl daemon-reload
    [root@tzgsqvelk01 tzg]# /bin/systemctl enable elasticsearch.service
    Created symlink from /etc/systemd/system/multi-user.target.wants/elasticsearch.service to /usr/lib/systemd/system/elasticsearch.service.
    [root@tzgsqvelk01 tzg]# systemctl start elasticsearch.service
    [root@tzgsqvelk01 tzg]# systemctl stop elasticsearch.service
    
    [root@tzgsqvelk01 tzg]# grep -v "^#" /etc/elasticsearch/elasticsearch.yml
    node.name: tzg-elk01
    path.data: /tzgData/elk/elasticsearch/data
    path.logs: /opt/elk/elasticsearch/logs
    network.host: 0.0.0.0
    
    [root@tzgsqvelk01 tzg]# mkdir -p /tzgData/elk/elasticsearch/data
    [root@tzgsqvelk01 tzg]# mkdir -p /opt/elk/elasticsearch/logs
    [root@tzgsqvelk01 tzg]# chown -R elasticsearch.elasticsearch /tzgData/elk/
    [root@tzgsqvelk01 tzg]# chown -R elasticsearch.elasticsearch /opt/elk/
    [root@tzgsqvelk01 tzg]#
    [root@tzgsqvelk01 tzg]# systemctl start elasticsearch.service
    [root@tzgsqvelk01 tzg]#
    [root@tzgsqvelk01 tzg]#
    [root@tzgsqvelk01 tzg]# curl http://localhost:9200
    {
      "name" : "tzg-elk01",
      "cluster_name" : "elasticsearch",
      "cluster_uuid" : "K8bEym6OSpm9o1Y4CJrrEg",
      "version" : {
        "number" : "5.1.1",
        "build_hash" : "5395e21",
        "build_date" : "2016-12-06T12:36:15.409Z",
        "build_snapshot" : false,
        "lucene_version" : "6.3.0"
      },
      "tagline" : "You Know, for Search"
    }
    
    
    [root@tzgsqvelk01 tzg]# systemctl stop elasticsearch.service
    [root@tzgsqvelk01 tzg]# /usr/share/elasticsearch/bin/elasticsearch-plugin install x-pack
    [root@tzgsqvelk01 tzg]# systemctl start elasticsearch.service
    [root@tzgsqvelk01 tzg]# curl -XPUT -u elastic 'localhost:9200/_xpack/security/user/elastic/_password' -d '{ "password" : "123456" }'
    Enter host password for user 'elastic':
    {}[root@tzgsqvelk01 tzg]#
    
    [root@tzgsqvelk01 tzg]# systemctl stop elasticsearch.service
    [root@tzgsqvelk01 tzg]# /usr/share/elasticsearch/bin/elasticsearch-plugin install x-pack
    [root@tzgsqvelk01 tzg]# systemctl start elasticsearch.service

#### kibana & x-pack安装 ####

[root@tzgsqvelk01 tzg]# cat > /etc/yum.repos.d/kibana.repo <<-EOF
    [kibana-5.x]
    name=Kibana repository for 5.x packages
    baseurl=https://artifacts.elastic.co/packages/5.x/yum
    gpgcheck=1
    gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
    enabled=1
    autorefresh=1
    type=rpm-md
    EOF
    [root@tzgsqvelk01 tzg]#
    [root@tzgsqvelk01 tzg]#
    [root@tzgsqvelk01 tzg]# yum install kibana -y
    
    [root@tzgsqvelk01 tzg]# grep "^[a-z]" /etc/kibana/kibana.yml
    server.host: "172.16.5.37"
    elasticsearch.username: "elastic"
    elasticsearch.password: "123456"
    
    [root@tzgsqvelk01 tzg]# systemctl stop kibana.service
    [root@tzgsqvelk01 tzg]# /usr/share/kibana/bin/kibana-plugin install file:///home/x-pack-5.1.1.zip
    [root@tzgsqvelk01 tzg]# systemctl start kibana.service
    [root@tzgsqvelk01 tzg]# curl -XPUT -u elastic 'localhost:9200/_xpack/security/user/kibana/_password' -d '{ 
      "password" : "123456"
    }' Enter host password for user 'elastic':

#### logstash 安装 ####

[root@tzgsqvelk01 tzg]# cat > /etc/yum.repos.d/logstash.repo <<-EOF
    [logstash-5.x]
    name=Elastic repository for 5.x packages
    baseurl=https://artifacts.elastic.co/packages/5.x/yum
    gpgcheck=1
    gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
    enabled=1
    autorefresh=1
    type=rpm-md
    EOF
    
    [root@tzgsqvelk01 tzg]# yum install logstash -y
    
    [root@tzgsqvelk01 tzg]# /usr/share/logstash/bin/logstash -e 'input { stdin{} } output { elasticsearch { hosts => "172.16.5.37:9200" user => elastic password => 123456 } } '
    hahahha
    WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
    Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs to console
    The stdin plugin is now waiting for input:
    10:47:39.397 [[main]-pipeline-manager] INFO  logstash.outputs.elasticsearch - Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>["http://~hidden~:~hidden~@172.16.5.37:9200"]}}
    10:47:39.400 [[main]-pipeline-manager] INFO  logstash.outputs.elasticsearch - Running health check to see if an Elasticsearch connection is working {:url=>#<URI::HTTP:0x673e5894 URL:http://~hidden~:~hidden~@172.16.5.37:9200>, :healthcheck_path=>"/"}
    10:47:39.617 [[main]-pipeline-manager] WARN  logstash.outputs.elasticsearch - Restored connection to ES instance {:url=>#<URI::HTTP:0x673e5894 URL:http://~hidden~:~hidden~@172.16.5.37:9200>}
    10:47:39.617 [[main]-pipeline-manager] INFO  logstash.outputs.elasticsearch - Using mapping template from {:path=>nil}
    10:47:39.658 [[main]-pipeline-manager] INFO  logstash.outputs.elasticsearch - Attempting to install template {:manage_template=>{
         "template"=>"logstash-*", "version"=>50001, "settings"=>{
         "index.refresh_interval"=>"5s"}, "mappings"=>{
         "_default_"=>{
         "_all"=>{
         "enabled"=>true, "norms"=>false}, "dynamic_templates"=>[{
         "message_field"=>{
         "path_match"=>"message", "match_mapping_type"=>"string", "mapping"=>{
         "type"=>"text", "norms"=>false}}}, {
         "string_fields"=>{
         "match"=>"*", "match_mapping_type"=>"string", "mapping"=>{
         "type"=>"text", "norms"=>false, "fields"=>{
         "keyword"=>{
         "type"=>"keyword"}}}}}], "properties"=>{
         "@timestamp"=>{
         "type"=>"date", "include_in_all"=>false}, "@version"=>{
         "type"=>"keyword", "include_in_all"=>false}, "geoip"=>{
         "dynamic"=>true, "properties"=>{
         "ip"=>{
         "type"=>"ip"}, "location"=>{
         "type"=>"geo_point"}, "latitude"=>{
         "type"=>"half_float"}, "longitude"=>{
         "type"=>"half_float"}}}}}}}}
    10:47:39.772 [[main]-pipeline-manager] INFO  logstash.outputs.elasticsearch - Installing elasticsearch template to _template/logstash
    10:47:39.815 [[main]-pipeline-manager] INFO  logstash.outputs.elasticsearch - New Elasticsearch output {:class=>"LogStash::Outputs::ElasticSearch", :hosts=>["172.16.5.37:9200"]} 10:47:39.817 [[main]-pipeline-manager] INFO logstash.pipeline - Starting pipeline {
         "id"=>"main", "pipeline.workers"=>8, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>5, "pipeline.max_inflight"=>1000}
    10:47:39.823 [[main]-pipeline-manager] INFO  logstash.pipeline - Pipeline main started
    10:47:39.856 [Api Webserver] INFO  logstash.agent - Successfully started Logstash API endpoint {:port=>9600}
    ^C10:47:47.587 [SIGINT handler] WARN  logstash.runner - SIGINT received. Shutting down the agent.
    10:47:47.592 [LogStash::Runner] WARN  logstash.agent - stopping pipeline {:id=>"main"}
    
    
    [root@tzgsqvelk01 tzg]# /usr/share/logstash/bin/system-install
    Successfully created system startup script for Logstash
    [root@tzgsqvelk01 tzg]# chown -R logstash.logstash /usr/share/logstash/data/

#### redis安装 ####

[root@ tzgsqvelk01 app]# wget http://download.redis.io/releases/redis-3.2.1.tar.gz
    [root@ tzgsqvelk01 app]# yum install gcc -y
    [root@ tzgsqvelk01 app]# tar zxvf redis-3.2.1.tar.gz
    [root@ tzgsqvelk01 app]# cd redis-3.2.1
    [root@tzgsqvelk01 redis-3.2.1]# make
    [root@tzgsqvelk01 redis-3.2.1]# make install PREFIX=/opt/app/redis
    
    [root@tzgsqvelk01 redis-3.2.1]# cat /opt/app/redis/conf/redis.conf
    daemonize yes pidfile /var/run/redis.pid bind 172.16.5.37 port 6379 timeout 120 loglevel notice logfile /opt/app/redis/logs/redis.log databases 16 save 900 1
    save 300 100 save 60 10000 stop-writes-on-bgsave-error yes rdbcompression yes rdbchecksum yes dbfilename dump.rdb dir /tzgData/redis/var appendonly yes appendfilename "appendonly.aof"
    appendfsync everysec no-appendfsync-on-rewrite no auto-aof-rewrite-percentage 100 auto-aof-rewrite-min-size 64mb aof-load-truncated yes slowlog-log-slower-than 10000 slowlog-max-len 128 hash-max-ziplist-entries 512 hash-max-ziplist-value 64 list-max-ziplist-entries 512 list-max-ziplist-value 64 set-max-intset-entries 512 zset-max-ziplist-entries 128 zset-max-ziplist-value 64 hll-sparse-max-bytes 3000 activerehashing yes maxmemory 4096000000 maxclients 10000 slave-serve-stale-data yes slave-read-only yes slave-priority 100 repl-diskless-sync no repl-diskless-sync-delay 5
    repl-disable-tcp-nodelay no 
    [root@tzgsqvelk01 redis-3.2.1]# cp utils/redis_init_script /etc/init.d/redisd
    [root@tzgsqvelk01 redis-3.2.1]# vim /etc/init.d/redisd
    
    #!/bin/sh
    # chkconfig: 2345 90 10
    # description: Redis is a persistent key-value database
    #
    
    REDISPORT=6379 EXEC=/opt/app/redis/bin/redis-server CLIEXEC=/opt/app/redis/bin/redis-cli 
    PIDFILE=/var/run/redis_${
         REDISPORT}.pid CONF="/opt/app/redis/conf/redis.conf"
    
    [root@tzgsqvelk01 tzg]# chkconfig redisd on
    [root@tzgsqvelk01 tzg]# service redisd start
    
    [root@tzgsqvelk01 app]# rm -rf redis-3.2.1
    [root@tzgsqvelk01 app]# rm -rf redis-3.2.1.tar.gz

#### logstash和elasticsearch 无需授权配置 ####

参考官方文档  
[https://www.elastic.co/guide/en/x-pack/current/anonymous-access.html][https_www.elastic.co_guide_en_x-pack_current_anonymous-access.html]

[root@ tzgsqvelk01 elasticsearch]# vim elasticsearch.yml
    
    xpack.security.authc:
      anonymous:
        username: anonymous_user
        roles: logstash_logs
        authz_exception: true
    
    ######添加 logstash_logs的权限######
    curl -XPOST -u elastic 'localhost:9200/_xpack/security/role/logstash_logs' -d '{
      "cluster": ["manage_index_templates", "monitor"],
      "indices": [
        {
          "names": [ "logstash-*","business-index-*"], 
          "privileges": ["write","delete","create_index"]
        }
      ]
    }'
    
    
    [root@ tzgsqvelk01 elasticsearch]# systemctl restart elasticsearch.service
    [root@ tzgsqvelk01 elasticsearch]# /usr/share/logstash/bin/logstash -e 'input { stdin{} } output { elasticsearch { hosts => "192.168.3.210:9200"} } '
    tzghahaha
    Sending Logstash's logs to /var/log/logstash/logs which is now configured via log4j2.properties
    The stdin plugin is now waiting for input:
    [2016-12-30T08:48:07,332][INFO ][logstash.outputs.elasticsearch] Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>["http://192.168.3.210:9200"]}}
    [2016-12-30T08:48:07,342][INFO ][logstash.outputs.elasticsearch] Running health check to see if an Elasticsearch connection is working {:url=>#<URI::HTTP:0x52cfa9fa URL:http://192.168.3.210:9200>, :healthcheck_path=>"/"}
    [2016-12-30T08:48:07,592][WARN ][logstash.outputs.elasticsearch] Restored connection to ES instance {:url=>#<URI::HTTP:0x52cfa9fa URL:http://192.168.3.210:9200>}
    [2016-12-30T08:48:07,599][INFO ][logstash.outputs.elasticsearch] Using mapping template from {:path=>nil}
    [2016-12-30T08:48:07,753][INFO ][logstash.outputs.elasticsearch] Attempting to install template {:manage_template=>{
         "template"=>"logstash-*", "version"=>50001, "settings"=>{
         "index.refresh_interval"=>"5s"}, "mappings"=>{
         "_default_"=>{
         "_all"=>{
         "enabled"=>true, "norms"=>false}, "dynamic_templates"=>[{
         "message_field"=>{
         "path_match"=>"message", "match_mapping_type"=>"string", "mapping"=>{
         "type"=>"text", "norms"=>false}}}, {
         "string_fields"=>{
         "match"=>"*", "match_mapping_type"=>"string", "mapping"=>{
         "type"=>"text", "norms"=>false, "fields"=>{
         "keyword"=>{
         "type"=>"keyword"}}}}}], "properties"=>{
         "@timestamp"=>{
         "type"=>"date", "include_in_all"=>false}, "@version"=>{
         "type"=>"keyword", "include_in_all"=>false}, "geoip"=>{
         "dynamic"=>true, "properties"=>{
         "ip"=>{
         "type"=>"ip"}, "location"=>{
         "type"=>"geo_point"}, "latitude"=>{
         "type"=>"half_float"}, "longitude"=>{
         "type"=>"half_float"}}}}}}}}
    [2016-12-30T08:48:07,773][INFO ][logstash.outputs.elasticsearch] New Elasticsearch output {:class=>"LogStash::Outputs::ElasticSearch", :hosts=>["192.168.3.210:9200"]}
    [2016-12-30T08:48:07,784][INFO ][logstash.pipeline        ] Starting pipeline {
         "id"=>"main", "pipeline.workers"=>4, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>5, "pipeline.max_inflight"=>500}
    [2016-12-30T08:48:07,814][INFO ][logstash.pipeline        ] Pipeline main started
    [2016-12-30T08:48:07,989][INFO ][logstash.agent           ] Successfully started Logstash API endpoint {:port=>9600}

[https_www.elastic.co_guide_en_elastic-stack_current_installing-elastic-stack.html]: https://www.elastic.co/guide/en/elastic-stack/current/installing-elastic-stack.html
[https_www.elastic.co_guide_en_x-pack_current_anonymous-access.html]: https://www.elastic.co/guide/en/x-pack/current/anonymous-access.html