filebeat.yml配置和logstash.conf

小咪咪 2022-07-12 02:59 173阅读 0赞

\#!/bin/bash  
  
  
/home/haoren/data/logstash-5.0.2/bin/logstash -f afactivityserver.conf &

(1)

filebeat.prospectors:

\- input\_type: log

paths:

- /log/abcbillserver.log

\#- /home/haoren/data/filebeat-5.0.2-linux-x86\_64/test.log

encoding: gbk

symlinks: true

include\_lines: \['\\\[.\*?统计\\\]','\\\[.\*?结算\\\]'\]

document\_type: billbijiesuan

fields\_under\_root: true

fields:

host: 192.168.10.7

processors:

\- drop\_fields:

\#fields: \["beat.hostname", "beat.name", "beat.version", "input\_type", "beat"\]

fields: \["input\_type", "beat", "offset", "source"\]

output.redis:

enabled: true

hosts: \["192.168.10.8"\]

password: "A8841c09BAD52E63067C4DA"

port: 6379

datatype: list

key: "filebeat"

db: 0

output.file:

enabled: false

path: "/tmp/filebeat"

output.console:

enabled: false

(2)

filebeat.prospectors:

\- input\_type: log

paths:

- /log/cactivityserver.log

\#- /home/haoren/data/filebeat-5.0.2-linux-x86\_64/test.log

encoding: gbk

symlinks: true

include\_lines: \['\\\[.\*?统计\\\]','\\\[.\*?结算\\\]'\]

document\_type: activityserver

fields\_under\_root: true

fields:

host: 192.168.10.13

\- input\_type: log

paths:

- /log/fvchannelserver.log

\#- /home/haoren/data/filebeat-5.0.2-linux-x86\_64/test.log

encoding: gbk

symlinks: true

include\_lines: \['\\\[.\*?统计\\\]','\\\[.\*?结算\\\]'\]

document\_type: vchannelserver

fields\_under\_root: true

fields:

host: 192.168.10.13

processors:

\- drop\_fields:

\#fields: \["beat.hostname", "beat.name", "beat.version", "input\_type", "beat"\]

fields: \["input\_type", "beat", "offset", "source"\]

output.redis:

enabled: true

hosts: \["192.168.10.8"\]

password: "A8841c09BAD52E63067C4DA"

port: 6379

datatype: list

key: "filebeat"

db: 0

output.file:

enabled: false

path: "/tmp/filebeat"

output.console:

enabled: false

logstash.conf 如下

input \{

redis \{

host => \["192.168.10.8"\]

port => 6379

password => "A8841c09BAD52E63067C4DA"

data\_type => "list"

key => "logstash"

codec => json \{

charset => "UTF-8"

filter \{

ruby \{

code=>"event.set('daytag',event.timestamp.time.localtime.strftime('%Y.%m.%d'))"

output \{

elasticsearch \{

hosts => \["127.0.0.1:19200"\]

index => "%\{type\}-%\{daytag\}"

\#index => "%\{type\}-%\{+yyyy.MM.dd\}"

user => "logstashserver"

password => "A950C0FB2D833E42C1AC59210CD5CDF8"

cat  /home/haoren/data/logstash-5.0.2/ billserver.conf  
  
input \{   
    file \{   
        path => "/log/abcbillserver.log"  
        codec => plain \{   
            charset => "GBK"  
        \}  
        \#start\_position => "beginning"  
        \#sincedb\_path => "/dev/null"  
        type => "billserver"  
    \}  
\}  
  
  
filter \{   
    if (\[message\] =~ "人民币统计")\{   
        mutate \{replace => \{ "type" => "dubijiesuan" \}\}  
        grok \{   
            \#170208-10:00:28 Bill\[40268\]  INFO: \[人民币结算\]时间(1486519228),用户ID(30581009),原来人民币(12567),现在人民币(16567),人民币操作(1),增加(4000),操作类型(19),操作详情(2),操作数量(0).描述:通用人民币操作  
            \#170208-10:00:01 Bill\[40268\]  INFO: \[人民币结算\]时间(1486519201),用户ID(22327945),原来人民币(2572),现在人民币(2532),人民币操作(2),扣除(40),操作类型(19),操作详情(15),操作数量(0).描述:通用人民币操作  
            match => \[ "message", "(?(?>\\d\{6\}-\\d\\d:\\d\\d:\\d\\d)) Bill\\\[\\d+\\\]  INFO: \\\[人民币结算\\\]时间\\(%\{NUMBER:time:int\}\\),用户ID\\(%\{NUMBER:userid:int\}\\),原来人民币\\(%\{NUMBER:oldcoin:int\}\\),现在人民币\\(%\{NUMBER:currentcoin:int\}\\),人民币操作\\(%\{NUMBER:coinop:int\}\\),%\{DATA:sop\}\\(%\{NUMBER:coinnum:int\}\\),操作类型\\(%\{NUMBER:optype:int\}\\),操作详情\\(%\{NUMBER:opdetail:int\}\\),操作数量\\(%\{NUMBER:opnum:int\}\\).描述:%\{DATA:sinfo\}"\]  
        \}  
    \}  
      else\{  
        drop\{\}  
    \}  
    date \{  
        match => \["datetime", "yyMMdd-HH:mm:ss"\]  
        \#timezone => \["Asia/Hong\_Kong"\]  
        \#remove\_field => \["time"\]  
    \}  
    ruby \{  
        code => "event.timestamp.time.localtime"  
    \}  
    mutate \{  
        \#some pc no host  
        replace => \{ "host" => "192.168.10.7" \}  
    \}  
\}  
  
  
output \{  
    \#stdout \{  
    \#    codec => plain \{  
    \#        charset => "UTF-8"  
    \#        \#charset => "GBK"  
    \#    \}  
    \#\}  
    \#file \{  
    \#    path => "/tmp/logstash.log"  
    \#    codec => json \{  
    \#        charset => "UTF-8"  
    \#    \}  
    \#\}  
    redis \{  
        host => \["192.168.10.18"\]  
        port => 6379  
        data\_type => "list"  
        key => "logstash"  
        password => "A8841c09BAD52E63067C4DA"  
        codec => json \{  
            charset => "UTF-8"  
        \}  
    \}  
\}  
  
cat  /home/haoren /data/logstash-5.0.2 / activityserver.conf  
  
input \{  
    file \{  
        path => "/log/aactivityserver.log"  
        codec => plain \{  
            charset => "GBK"  
        \}  
        \#start\_position => "beginning"  
        \#sincedb\_path => "/dev/null"  
        type => "aactivityserver"  
    \}  
\}  
  
  
filter \{  
    if (\[message\] =~ "运统计")\{  
        mutate \{replace => \{ "type" => "activityescort" \}\}  
        grok \{  
            \#161201-13:12:28 ActivityServer\[17701\]  INFO: \[Escort.cpp:595\] \[统计\]序号(53)用户(23619530)(攻)值(4360)暴击率(4)使用道具(57)本次花费(0)本总花费(0)车原始量(1706792)剩余量(1702432)总值(4360)  
            match => \[ "message", "(?(?>\\d\{6\}-\\d\\d:\\d\\d:\\d\\d)) ActivityServer\\\[\\d+\\\].\*?INFO: \\\[.\*?\\\] \\\[运镖统计\\\]序号\\(%\{NUMBER:carid:int\}\\)用户\\(%\{NUMBER:userid:int\}\\)\\(%\{DATA:sop\}\\).\*?\\(%\{NUMBER:itemvalue:int\}\\)暴击率\\(%\{NUMBER:baoji:int\}\\)使用道具\\(%\{NUMBER:itemid:int\}\\)本次花费\\(%\{NUMBER:coin:int\}\\)本总花费\\(%\{NUMBER:allusercoin:int\}\\)车原始量\\(%\{NUMBER:oldblood:int\}\\)剩余量\\(%\{NUMBER:blood:int\}\\)总值\\(%\{NUMBER:allvalue:int\}\\)"\]  
        \}  
    \}  
    else if (\[message\] =~ "运通用奖励")\{  
        mutate \{replace => \{ "type" => "activityescort" \}\}  
        grok \{  
            \#161201-13:15:16 ActivityServer\[17701\]  INFO: \[运\]序号(54), \[运通用奖励\] 用户(21772699)奖励方式(0)奖励类型(34)购物车ID(61)数量(16)  
            match => \[ "message", "(?(?>\\d\{6\}-\\d\\d:\\d\\d:\\d\\d)) ActivityServer\\\[\\d+\\\]  INFO: \\\[运\\\]序号\\(%\{NUMBER:carid:int\}\\), \\\[运通用奖励\\\] 用户\\(%\{NUMBER:userid:int\}\\)奖励方式\\(%\{NUMBER:method:int\}\\)奖励类型\\(%\{NUMBER:addtype:int\}\\)包裹ID\\(%\{NUMBER:itemid:int\}\\)数量\\(%\{NUMBER:num:int\}\\)"\]  
        \}  
    \}  
    else\{  
        drop\{\}  
    \}  
    date \{  
        match => \["datetime", "yyMMdd-HH:mm:ss"\]  
        \#timezone => \["Asia/Hong\_Kong"\]  
        \#remove\_field => \["time"\]  
    \}  
    ruby \{  
        code => "event.timestamp.time.localtime"  
    \}  
    mutate \{  
        \#some pc no host  
        replace => \{ "host" => "192.168.10.13" \}  
    \}  
\}  
  
  
output \{  
    \#stdout \{  
    \#    codec => plain \{  
    \#        charset => "UTF-8"  
    \#        \#charset => "GBK"  
    \#    \}  
    \#\}  
    \#file \{  
    \#    path => "/tmp/logstash.log"  
    \#    codec => json \{  
    \#        charset => "UTF-8"  
    \#    \}  
    \#\}  
    redis \{  
        host => \["192.168.10.8"\]  
        port => 6379  
        data\_type => "list"  
        key => "logstash"  
        password => "A8841c09BAD52E63067C4DA"  
        codec => json \{  
            charset => "UTF-8"  
        \}  
    \}  
\}  
  
  
  
cat /home/haoren/data/filebeat-5.0.2-linux-x86\_64/filebeat.yml    
  
  
filebeat.prospectors:  
\- input\_type: log  
  paths:  
    - /log/billserver.log  
    \#- /home/haoren/data/filebeat-5.0.2-linux-x86\_64/test.log  
  encoding: gbk  
  symlinks: true  
  include\_lines: \['\\\[.\*?统计\\\]','\\\[.\*?结算\\\]'\]  
  document\_type: billserver  
  fields\_under\_root: true  
  fields:  
    host: 192.168.10.7  
  
  
processors:  
\- drop\_fields:  
    \#fields: \["beat.hostname", "beat.name", "beat.version", "input\_type", "beat"\]  
    fields: \["input\_type", "beat", "offset", "source"\]  
  
  
output.redis:  
  enabled: true  
  hosts: \["192.168.10.18"\]  
  password: "A8841c09BAD52E63067C4DA"  
  port: 6379  
  datatype: list  
  key: "filebeat"  
  db: 0  
  
  
output.file:  
  enabled: false  
  path: "/tmp/filebeat"  
  
  
output.console:  
  enabled: false