Mysql+Flume+Kafka 雨点打透心脏的1/2处 2022-05-20 03:19 164阅读 0赞 下载编译flume-ng-sql-source,[下载地址][Link 1]并按照网站里的说明添加相应的包 安装说明文档编译和拷贝jar包 flume配置文件 a1.channels = c1 c2 a1.sources = s1 s2 a1.sinks = k1 k2 ###########sources################# # s1 a1.sources.s1.type = org.keedio.flume.source.SQLSource a1.sources.s1.hibernate.connection.url = jdbc:mysql://192.168.1.177:3306/bigdata a1.sources.s1.hibernate.connection.user = root a1.sources.s1.hibernate.connection.password = chineseall a1.sources.s1.hibernate.connection.autocommit = true a1.sources.s1.hibernate.dialect = org.hibernate.dialect.MySQL5Dialect a1.sources.s1.hibernate.connection.driver_class = com.mysql.jdbc.Driver a1.sources.s1.run.query.delay=10000 a1.sources.s1.status.file.path = /root/data/flume/ a1.sources.s1.status.file.name = sqlSource.status a1.sources.s1.start.from = 0 a1.sources.s1.custom.query = select username,devicetype,logintime from device_log a1.sources.s1.batch.size = 1000 a1.sources.s1.max.rows = 1000 a1.sources.s1.hibernate.connection.provider_class = org.hibernate.connection.C3P0ConnectionProvider a1.sources.s1.hibernate.c3p0.min_size=1 a1.sources.s1.hibernate.c3p0.max_size=10 #s2 a1.sources.s2.type = org.keedio.flume.source.SQLSource a1.sources.s2.hibernate.connection.url = jdbc:mysql://192.168.1.177:3306/bigdata a1.sources.s2.hibernate.connection.user = root a1.sources.s2.hibernate.connection.password = chineseall a1.sources.s2.hibernate.connection.autocommit = true a1.sources.s2.hibernate.dialect = org.hibernate.dialect.MySQL5Dialect a1.sources.s2.hibernate.connection.driver_class = com.mysql.jdbc.Driver a1.sources.s2.run.query.delay=10000 a1.sources.s2.status.file.path = /root/data/flume/ a1.sources.s2.status.file.name = sqlSource.status a1.sources.s2.start.from = 0 a1.sources.s2.custom.query = select realname,school,source,create_time from report_table a1.sources.s2.batch.size = 1000 a1.sources.s2.max.rows = 1000 a1.sources.s2.hibernate.connection.provider_class = org.hibernate.connection.C3P0ConnectionProvider a1.sources.s2.hibernate.c3p0.min_size=1 a1.sources.s2.hibernate.c3p0.max_size=10 ############channels############### a1.channels.c1.type = memory a1.channels.c1.capacity = 10000 a1.channels.c1.transactionCapacity = 10000 a1.channels.c1.byteCapacityBufferPercentage = 20 a1.channels.c1.byteCapacity = 800000 a1.channels.c2.type = memory a1.channels.c2.capacity = 10000 a1.channels.c2.transactionCapacity = 10000 a1.channels.c2.byteCapacityBufferPercentage = 20 a1.channels.c2.byteCapacity = 800000 ############sinks################## a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink a1.sinks.k1.topic = Device a1.sinks.k1.brokerList = 192.168.1.233:9092 a1.sinks.k1.requiredAcks = 1 a1.sinks.k1.batchSize = 20 a1.sinks.k1.channel = c1 a1.sinks.k2.type = org.apache.flume.sink.kafka.KafkaSink a1.sinks.k2.topic = Report a1.sinks.k2.brokerList = 192.168.1.233:9092 a1.sinks.k2.requiredAcks = 1 a1.sinks.k2.batchSize = 20 a1.sinks.k2.channel = c1 # Bind the source and sink to the channel a1.sinks.k1.channel = c1 a1.sinks.k2.channel = c2 a1.sources.s1.channels=c1 a1.sources.s2.channels=c2 [Link 1]: https://github.com/keedio/flume-ng-sql-source
还没有评论，来说两句吧...