将nginx的日志通过logstash载入到ES中logstash.cnof配置

怼烎@ 2022-05-21 07:40 426阅读 0赞

1、nginx日志格式化为json格式的配置（nginx日志json格式化请自行google）：

input \{  \# 定义日志源  
\# file \{   \#从文件中拉取  
   \#    type => "log"  
   \#    path => "D:/log/academy\_\*.log"  
   \#    start\_position => "beginning"   
   \# \}

beats \{  \#从fileBeat中获取  
          port => "5044"  
   \}

\# kafka \{  \#从kafka中获取  
\#       bootstrap\_servers => "127.0.0.1:9092"  
\#       topics => \["ecplogs"\]  
\#   \}

filter \{

json \{ \#日志为json格式  
       source => "message"  
       \#remove\_field => \["message"\]  
  \}  
  date \{  \#日志中包括timestamp字段  
       match => \[ "timestamp", "UNIX", "ISO8601", "yyyy-MM-dd HH:mm:ss" \]  
  \}  
   
\}

output \{  \# 定义日志输出

elasticsearch \{

hosts => \["localhost:9200"\]       \# 定义es服务器的ip  
        index => "logstash-system-syslog-%\{+YYYY.MM.dd\}"   \# 定义索引名称  
    \}

\#    stdout \{  
\#     codec => rubydebug \{ \}  
\#    \}

2、nginx使用默认格式化形式:

input \{  \# 定义日志源  
  \# file \{  
  \#     type => "log"  
  \#     path => "D:/nginx-1.14.0/access.log"  
  \#     \#start\_position => "beginning"   
  \#  \}

beats \{  
       port => "5044"  
   \}

\#   stdin \{ \}

\#   kafka \{  
\#       bootstrap\_servers => "127.0.0.1:9092"  
\#       topics => \["ecplogs"\]  
\#   \}

filter \{

grok \{  
      match => \{ "message" => "%\{IP:clientIp\} - - \\\[%\{HTTPDATE:logdate\}\\\] \\"%\{WORD:verb\} %\{URIPATHPARAM:request\} HTTP/%\{NUMBER:httpversion\}\\" %\{NUMBER:http\_status\_code\} %\{NUMBER:bytes\} \\"%\{NOTSPACE:ref\}\\" \\"%\{DATA:user\_agent\}\\"" \}  
      \}

date \{  
         match => \[ "logdate", "UNIX", "ISO8601", "dd/MMM/yyyy:HH:mm:ss Z","yyyy-MM-dd HH:mm:ss" \]  
  target => "@timestamp"  
       \}

kv  \{  
          source => "request"  
          field\_split => "&?"  
          value\_split => "="

urldecode \{

all\_fields => true

geoip \{ source => "clientIp" \}

output \{  \# 定义日志输出

elasticsearch \{  
        hosts => \["localhost:9200"\]       \# 定义es服务器的ip  
        index => "logstash-nginx-%\{+YYYY.MM.dd\}"   \# 定义索引名称  
    \}

stdout \{  
        codec => json\_lines  
    \}