One way: read the file output to Kafka
1. Code
import org.apache.flink.api.common.serialization.SimpleStringSchema
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer011
//温度传感器读取样例类
case class SensorReading(id: String, timestamp: Long, temperature: Double)
object KafkaSinkTest {
def main(args: Array[String]): Unit = {
val env = StreamExecutionEnvironment.getExecutionEnvironment
env.setParallelism(1)
import org.apache.flink.api.scala._
val inputStream = env.readTextFile("sensor.txt")
val dataStream = inputStream.map(x => {
val arr = x.split(",")
SensorReading (arr (0) .trim, arr (1) .trim.toLong, arr (2) .trim.toDouble) .toString // String easily converted into a sequence of output
})
// sink
dataStream.addSink (new new FlinkKafkaProducer011 [ String] ( "localhost: 9092", "sinkTest", new new SimpleStringSchema ()))
dataStream.print ()
env.execute ( "sink Kafka Test")
}
}
2. start zookeeper: reference https: //www.cnblogs. COM / wddqy / the p-/ 12156527.html
3. start kafka: reference https://www.cnblogs.com/wddqy/p/12156527.html
4. create a kafka consumers observations
Second way: Kafka to Kafka
1. Code
import java.util.Properties
import org.apache.flink.api.common.serialization.SimpleStringSchema
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
import org.apache.flink.streaming.connectors.kafka.{FlinkKafkaConsumer011, FlinkKafkaProducer011}
//温度传感器读取样例类
case class SensorReading(id: String, timestamp: Long, temperature: Double)
object KafkaSinkTest1 {
def main(args: Array[String]): Unit = {
val env = StreamExecutionEnvironment.getExecutionEnvironment
env.setParallelism(1)
import org.apache.flink.api.scala._
//从Kafka到Kafka
val properties = new Properties()
properties.setProperty("bootstrap.servers", "localhost:9092")
properties.setProperty("group.id", "consumer-group")
properties.setProperty("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer")
properties.setProperty("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer")
properties.setProperty("auto.offset.reset", "latest")
val inputStream = env.addSource(new FlinkKafkaConsumer011[String]("sensor", new SimpleStringSchema(), properties))
val dataStream = inputStream.map(x => {
val arr = x.split(",")
SensorReading(arr(0).trim, arr(1).trim.toLong, arr(2).trim.toDouble).toString //转成String方便序列化输出
})
//sink
dataStream.addSink(new FlinkKafkaProducer011[String]("localhost:9092", "sinkTest", new SimpleStringSchema()))
dataStream.print()
env.execute(" kafka sink test")
}
}
2. Start zookeeper: Reference https://www.cnblogs.com/wddqy/p/12156527.html
3. Start kafka: Reference https://www.cnblogs.com/wddqy/p/12156527.html
4. Create a Kafka producers and consumers, run the code, observations