2. Flume Source 详解

喜欢ヅ旅行 2022-06-14 00:24 235阅读 0赞

## 1.  Avro Source属性说明 ##

a1.sources = r1
    a1.channels = c1
    a1.sources.r1.type = avro
    a1.sources.r1.channels = c1
    a1.sources.r1.bind = 0.0.0.0
    a1.sources.r1.port = 4141

## 2. Spooling Directory Source ##

监测配置的目录下新增的文件，并将文件中的数据读取出来。

a1.channels = ch-1
    a1.sources = src-1
    
    a1.sources.src-1.type = spooldir
    a1.sources.src-1.channels = ch-1
    a1.sources.src-1.spoolDir = /var/log/apache/flumeSpool
    a1.sources.src-1.fileHeader = true

## 3.    NetCat Source ##

一个NetCat Source用来监听一个指定端口，并将接收到的数据的每一行转换为一个事件。

###   NetCat Source属性说明 ###

![Center][]

## 4.    HTTP Source ##

HTTP Source接受HTTP的GET和POST请求作为Flume的事件,其中GET方式应该只用于试验。

a1.sources = r1
    a1.channels = c1
    a1.sources.r1.type = http
    a1.sources.r1.port = 5140
    a1.sources.r1.channels = c1
    a1.sources.r1.handler = org.example.rest.RestHandler
    a1.sources.r1.handler.nickname = random props

###  ###

## 5. Kafka Source    ##

## Example for topic subscription by comma-separated topic list.  tier1.sources.source1.type **=** org.apache.flume.source.kafka.KafkaSource tier1.sources.source1.channels **=** channel1 tier1.sources.source1.batchSize **=** 5000 tier1.sources.source1.batchDurationMillis **=** 2000 tier1.sources.source1.kafka.bootstrap.servers **=** localhost:9092 tier1.sources.source1.kafka.topics **=** test1, test2 tier1.sources.source1.kafka.consumer.group.id **=** custom.g.id  Example for topic subscription by regex  tier1.sources.source1.type **=** org.apache.flume.source.kafka.KafkaSource tier1.sources.source1.channels **=** channel1 tier1.sources.source1.kafka.bootstrap.servers **=** localhost:9092 tier1.sources.source1.kafka.topics.regex **=** ^topic\[0-9\]$ *\# the default kafka.consumer.group.id=flume is used*      ##

## 6.    Thrift Source ##

a1.sources = r1
    a1.channels = c1
    a1.sources.r1.type = thrift
    a1.sources.r1.channels = c1
    a1.sources.r1.bind = 0.0.0.0
    a1.sources.r1.port = 4141

## 7.    Exec Source ##

tail -f 命令监控

a1.sources = r1
    a1.channels = c1
    a1.sources.r1.type = exec
    a1.sources.r1.command = tail -F /var/log/secure
    a1.sources.r1.channels = c1

**8.JMS Source**

**9.TailDir Source**

a1.sources = r1
    a1.channels = c1
    a1.sources.r1.type = TAILDIR
    a1.sources.r1.channels = c1
    a1.sources.r1.positionFile = /var/log/flume/taildir_position.json
    a1.sources.r1.filegroups = f1 f2
    a1.sources.r1.filegroups.f1 = /var/log/test1/example.log
    a1.sources.r1.headers.f1.headerKey1 = value1
    a1.sources.r1.filegroups.f2 = /var/log/test2/.*log.*
    a1.sources.r1.headers.f2.headerKey1 = value2
    a1.sources.r1.headers.f2.headerKey2 = value2-2
    a1.sources.r1.fileHeader = true

**10.SySLog  Source**

a1.sources = r1
    a1.channels = c1
    a1.sources.r1.type = syslogtcp
    a1.sources.r1.port = 5140
    a1.sources.r1.host = localhost
    a1.sources.r1.channels = c1

**11.MULTIPORT SysLog TCP  Source**

a1.sources = r1
    a1.channels = c1
    a1.sources.r1.type = multiport_syslogtcp
    a1.sources.r1.channels = c1
    a1.sources.r1.host = 0.0.0.0
    a1.sources.r1.ports = 10001 10002 10003
    a1.sources.r1.portHeader = port

**12.SySLog  UDP Source**

a1.sources = r1
    a1.channels = c1
    a1.sources.r1.type = syslogudp
    a1.sources.r1.port = 5140
    a1.sources.r1.host = localhost
    a1.sources.r1.channels = c1

13.Custom Source

a1.sources = r1
    a1.channels = c1
    a1.sources.r1.type = org.example.MySource
    a1.sources.r1.channels = c1

14.Scribe Source

a1.sources = r1
    a1.channels = c1
    a1.sources.r1.type = org.apache.flume.source.scribe.ScribeSource
    a1.sources.r1.port = 1463
    a1.sources.r1.workerThreads = 5
    a1.sources.r1.channels = c1

[Center]: /images/20220614/d24204a527dc423f9e5b3fa17f380935.png