kafka 소개 :
kafka 개념 : https://blog.51cto.com/mapengfei/1926063
kafka 설치 : https://blog.51cto.com/mapengfei/1926065
소비자 및 생산자 데모 : https : // blog. 51cto.com/mapengfei/1926068
Flink 출력 kafka 샘플은 주로 txt 파일에서 읽고 kafka에 쓰는 것입니다.
package com.mafei.sinktest
import org.apache.flink.api.common.serialization.SimpleStringSchema
import org.apache.flink.streaming.api.scala.{StreamExecutionEnvironment, createTypeInformation}
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer011
case class SensorReadingTest4(id: String,timestamp: Long, temperature: Double)
object KafkaSink {
def main(args: Array[String]): Unit = {
//创建执行环境
val env = StreamExecutionEnvironment.getExecutionEnvironment
val inputStream= env.readTextFile("/opt/java2020_study/maven/flink1/src/main/resources/sensor.txt")
env.setParallelism(1)
//先转换成样例类类型
val dataStream = inputStream
.map(data =>{
val arr = data.split(",") //按照,分割数据,获取结果
SensorReadingTest4(arr(0), arr(1).toLong,arr(2).toDouble).toString //生成一个传感器类的数据,参数中传toLong和toDouble是因为默认分割后是字符串类别
})
dataStream.addSink(new FlinkKafkaProducer011[String]("localhost:9092", "sensor", new SimpleStringSchema()))
dataStream.print()
env.execute()
}
}
서버의 명령 줄에서 소비자를 시작하여 데이터를 사용할 수 있습니다.
[root@localhost ~]# sh /opt/kafka_2.11-0.10.2.0/bin/kafka-console-consumer.sh --bootstrap-server 127.0.0.1:9092 --topic sensor --consumer.config /opt/kafka_2.11-0.10.2.0/config/consumer.properties
[2020-11-02 20:08:51,679] WARN The configuration 'zookeeper.connect' was supplied but isn't a known config. (org.apache.kafka.clients.consumer.ConsumerConfig)
[2020-11-02 20:08:51,683] WARN The configuration 'zookeeper.connection.timeout.ms' was supplied but isn't a known config. (org.apache.kafka.clients.consumer.ConsumerConfig)
SensorReadingTest4(sensor3,1603766283,43.0)
SensorReadingTest4(sensor4,1603766240,40.1)
SensorReadingTest4(sensor1,1603766281,41.0)
SensorReadingTest4(sensor2,1603766282,42.0)
SensorReadingTest4(sensor4,1603766249,40.2)
SensorReadingTest4(sensor4,1603766284,20.0)
코드 구조