Flume 收集Nginx日志 tail-to-avro avro-to-file

约定不等于承诺〃 2022-08-26 04:49 145阅读 0赞

### nginx，access.log日志约1.6w条/s，每1000w条数据约2000M,需要10min ###

### tail-to-avro ###

agent1.sources = source1
    agent1.sinks = sink1
    agent1.channels = channel1
    
    # Describe/configure spooldir source1
    #agent1.sources.source1.type = spooldir
    #agent1.sources.source1.spoolDir = /var/log/apache/flumeSpool1
    #agent1.sources.source1.fileHeader = true
    
    # Describe/configure tail -F source1
    agent1.sources.source1.type = exec 
    agent1.sources.source1.command = tail -F /tmp/log.log
    agent1.sources.source1.channels = channel1
    
    # Describe/configure nc source1
    #agent1.sources.source1.type = netcat
    #agent1.sources.source1.bind = localhost
    #agent1.sources.source1.port = 44444
    
    #configure host for source
    agent1.sources.source1.interceptors = i1
    agent1.sources.source1.interceptors.i1.type = host
    agent1.sources.source1.interceptors.i1.hostHeader = hostname
    
    # Describe logger sink1
    #agent1.sinks.sink1.type = logger
    
    # Describe avro sink1
    agent1.sinks.sink1.type = avro
    agent1.sinks.sink1.hostname = 172.16.10.175
    agent1.sinks.sink1.port = 4545
    
    # Use a channel which buffers events in memory
    agent1.channels.channel1.type = memory
    agent1.channels.channel1.keep-alive = 120
    agent1.channels.channel1.capacity = 500000
    agent1.channels.channel1.transactionCapacity = 600
    
    # Bind the source and sink to the channel
    agent1.sources.source1.channels = channel1
    agent1.sinks.sink1.channel = channel1
      
       
    
      
       
      
       
    
      
       
      
       
       
        avro-to-rollfile
       
        agent1.sources = source1
    agent1.sinks = sink1
    agent1.channels = channel1
    
    # Describe/configure avro source
    
    agent1.sources.source1.type = avro
    agent1.sources.source1.bind = 0.0.0.0
    agent1.sources.source1.port = 4545
    
    
    # Describe logger sink1
    #agent1.sinks.sink1.type = logger
    
    # Describe file sink1
    agent1.sinks.sink1.type = file_roll
    agent1.sinks.sink1.sink.directory = /var/log/flume
    
    # Use a channel which buffers events in memory
    agent1.channels.channel1.type = memory
    agent1.channels.channel1.keep-alive = 120
    agent1.channels.channel1.capacity = 500000
    agent1.channels.channel1.transactionCapacity = 600
    
    # Bind the source and sink to the channel
    agent1.sources.source1.channels = channel1
    agent1.sinks.sink1.channel = channel1