Flume的source、channel，sink简单介绍及使用

Source

# source-spooldir    目錄一經監聽就不能再做修改，也不能再重命名
a1.sources=r1
a1.channels=c1
a1.sinks=s1

a1.sources.r1.type=spooldir
a1.sources.r1.spoolDir=/root/data/test


a1.channels.c1.type=memory
a1.channels.c1.capacity=100
a1.channels.c1.transactionCapacity=100

a1.sinks.s1.type=logger


a1.sources.r1.channels=c1
a1.sinks.s1.channel=c1

#source-HTTP

a1.sources=r1
a1.channels=c1
a1.sinks=s1

a1.sources.r1.type=HTTP
a1.sources.r1.port=4444

a1.channels.c1.type=memory
a1.channels.c1.capacity=100
a1.channels.c1.transactionCapacity=100

a1.sinks.s1.type=logger


a1.sources.r1.channels=c1
a1.sinks.s1.channel=c1



#用於測試的json
curl -X POST -d '[{"headers":{"a":"a1","b":"b1"},"body":"hello http-flume"}]'  http://0.0.0.0:4444（4444是port端口號，根據情況修改）

#source-netcat

a1.sources=r1
a1.channels=c1
a1.sinks=s1

a1.sources.r1.type=netcat
a1.sources.r1.port=4444
a1.sources.r1.bind=0.0.0.0

a1.channels.c1.type=memory
a1.channels.c1.capacity=100
a1.channels.c1.transactionCapacity=100

a1.sinks.s1.type=logger


a1.sources.r1.channels=c1
a1.sinks.s1.channel=c1

Channel

#channel-kafka
a1.sources=r1
a1.channels=c1
a1.sinks=s1

a1.sources.r1.type=netcat
a1.sources.r1.port=4444
a1.sources.r1.bind=0.0.0.0

a1.channels.c1.type=org.apache.flume.channel.kafka.KafkaChannel
a1.channels.c1.capacity=1000
a1.channels.c1.transactionCapacity=100
a1.channels.c1.brokerList=node132:9092
a1.channels.c1.topic=1705a
a1.channels.c1.zookeeperConnect=node132:2181
a1.channels.c1.parseAsFlumeEvent=false

a1.sinks.s1.type=logger


a1.sources.r1.channels=c1
a1.sinks.s1.channel=c1

# channel-file    存到指定目錄下
a1.sources=r1
a1.channels=c1
a1.sinks=s1

a1.sources.r1.type=netcat
a1.sources.r1.bind=0.0.0.0
a1.sources.r1.port=4444

a1.channels.c1.type=file
a1.channels.c1.dataDirs=/root/data/test

a1.sinks.s1.type=logger


a1.sources.r1.channels=c1
a1.sinks.s1.channel=c1

# channel-memory  存到内存中
a1.sources=r1
a1.channels=c1
a1.sinks=s1

a1.sources.r1.type=netcat
a1.sources.r1.port=4444
a1.sources.r1.bind=0.0.0.0

a1.channels.c1.type=memory
a1.channels.c1.capacity=100
a1.channels.c1.transactionCapacity=100

a1.sinks.s1.type=logger


a1.sources.r1.channels=c1
a1.sinks.s1.channel=c1

Sink

# sink-kafka
a1.sources=r1
a1.channels=c1
a1.sinks=s1


a1.sources.r1.type=exec
a1.sources.r1.command=tail -F /root/data/log

a1.channels.c1.type=memory
a1.channles.c1.capacity=1000
a1.channles.c1.transactionCapacity=200

# a1.sinks.s1.type = org.apache.flume.sink.kafka.KafkaSink
# a1.sinks.s1.topic = 1705a
# a1.sinks.s1.brokerList = node132:9092
# a1.sinks.s1.requiredAcks = 1
# a1.sinks.s1.batchSize = 20
a1.sinks.s1.type = logger


a1.sources.r1.channels=c1
a1.sinks.s1.channel = c1

# sink-file_roll   将接收到的内容保存到指定目录下
a1.sources=r1
a1.channels=c1
a1.sinks=s1

a1.sources.r1.type=netcat
a1.sources.r1.bind=0.0.0.0
a1.sources.r1.port=4444


a1.channels.c1.type=memory
a1.channels.c1.capacity=100
a1.channels.c1.transactionCapacity=100

a1.sinks.s1.type=file_roll 
a1.sinks.s1.sink.directory=/root/data/test
#每60秒生成一个新的日志文件，设置成0则接收到的所有东西都存到一个文件中
a1.sinks.s1.sink.rollInterval=60   


a1.sources.r1.channels=c1
a1.sinks.s1.channel=c1

sink-avro 多级流动形式

#sink-hdfs
a1.sources=r1
a1.channels=c1
a1.sinks=s1

a1.sources.r1.type=netcat
a1.sources.r1.bind=0.0.0.0
a1.sources.r1.port=4444


a1.channels.c1.type=memory
a1.channels.c1.capacity=100
a1.channels.c1.transactionCapacity=100

a1.sinks.s1.type=hdfs
a1.sinks.s1.hdfs.path=hdfs://node132:9000/flume
#多久生成一个新的文件（秒）
a1.sinks.s1.hdfs.rollInterval=30
#文件多大之后生成新的文件（字节）
a1.sinks.s1.hdfs.rollSize=1024
#文件个数达到多少之后生成新的文件
a1.sinks.s1.hdfs.rollCount=10
a1.sinks.s1.hdfs.fileType=DataStream

a1.sources.r1.channels=c1
a1.sinks.s1.channel=c1