/**
* This is a code sample written with FlinkCEP 1.3.2. It is not meant to be compiled on its own,
* but you can follow the steps in the quickstart guide
* (https://ci.apache.org/projects/flink/flink-docs-release-1.3/quickstart/java_api_quickstart.html)
* to create a Flink program .
*/
package com.dataartisans.flinktraining.exercises.cep;
import com.dataartisans.flinktraining.exercises.custom.TaxiRideWithTimestamp;
import com.dataartisans.flinktraining.exercises.datastream_java.datatypes.TaxiRide;
import com.dataartisans.flinktraining.exercises.datastream_java.utils.TaxiRideSchema;
import com.dataartisans.flinktraining.exercises.datastream_java.utils.TransportSchema;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.utils.ParameterTool;
import org.apache.flink.cep.CEP;
import org.apache.flink.cep.PatternFlatSelectFunction;
import org.apache.flink.cep.pattern.Pattern;
import org.apache.flink.cep.pattern.conditions.IterativeCondition;
import org.apache.flink.cep.pattern.conditions.SimpleCondition;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.TimerService;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011;
import org.apache.flink.util.Collector;
import java.util.*;
public class RetailerExampleKafka {
private static final String LOCAL_KAFKA_BROKER = "localhost:9092";
// private static final String RIDE_SPEED_GROUP = "rideSpeedGroup";
/**
* The initial source of our shipment.
*/
private static final String SRC = "a";
/**
* The final destination of our shipment.
*/
private static final String DST = "h";
/**
* The required intermediate stops.
*/
private static final int NUM_STOPS = 5;
/**
* A pattern that searches for shipments that:
* 1) start from SRC
* 2) finish in DST
* 3) have at least NUM_STOPS stops in-between.
*/
private static final Pattern<Transport, ?> pattern = Pattern.<Transport>begin("start")
.where(new SimpleCondition<Transport>() {
private static final long serialVersionUID = 314415972814127035L;
@Override
public boolean filter(Transport value) throws Exception {
return Objects.equals(value.getFrom(), SRC);
}
}).next("middle").where(new SimpleCondition<Transport>() {
private static final long serialVersionUID = 6664468385615273240L;
@Override
public boolean filter(Transport value) {
return value.getFrom().startsWith("b");
}
}).next("end").where(new SimpleCondition<Transport>() {
private static final long serialVersionUID = 5721311694340771858L;
@Override
public boolean filter(Transport value) {
return value.getFrom().startsWith("c");
}
}).within(Time.milliseconds(5000));
public static void main(String[] args) throws Exception {
List<Transport> sampleData = new ArrayList<>();
sampleData.add(new Transport(1, "a", "b", 0L));
sampleData.add(new Transport(1, "b", "c", 1L));
sampleData.add(new Transport(1, "c", "d", 2L));
// StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
env.setBufferTimeout(5);
env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
env.getConfig().disableSysoutLogging();
/* ParameterTool params = ParameterTool.fromArgs(args);
String inputTopic = params.getRequired("input-topic");
String groupID = params.getRequired("group-id");
Long slide = Long.parseLong(params.getRequired("slide").trim());
*/
String inputTopic = "flinkceptest40";
String groupID = "groupID12131";
final int popThreshold = 1; // threshold for popular places
env.getConfig().setAutoWatermarkInterval(1000);
Properties kafkaProps = new Properties();
//kafkaProps.setProperty("zookeeper.connect", LOCAL_ZOOKEEPER_HOST);
kafkaProps.setProperty("bootstrap.servers", LOCAL_KAFKA_BROKER);
kafkaProps.setProperty("group.id", "g111");
kafkaProps.setProperty("auto.offset.reset", "earliest");
// create a Kafka consumer
FlinkKafkaConsumer011<Transport> consumer = new FlinkKafkaConsumer011<>(
inputTopic,
new TransportSchema(),
kafkaProps);
DataStream<Transport> rides = env.addSource(consumer)
.keyBy(element -> element.getProductId())
.process(new MatchFunction2());
CEP.pattern(rides, pattern).flatSelect(new PatternFlatSelectFunction<Transport, String>() {
private static final long serialVersionUID = -8972838879934875538L;
@Override
public void flatSelect(Map<String, List<Transport>> map, Collector<String> collector) throws Exception {
StringBuilder str = new StringBuilder();
for (Map.Entry<String, List<Transport>> entry : map.entrySet()) {
for (Transport t : entry.getValue()) {
t.timestamp = System.currentTimeMillis();
str.append(t + " ");
}
}
collector.collect(str.toString());
}
}).print();
env.execute();
}
/**
* Our input records. Each contains:
* 1. the id of the product,
* 2. the starting location of the shipment, and
* 3. the final location of the shipment.
*/
public static class Transport {
private final int prodId;
private final String from;
private final String to;
private long timestamp;
public long ingestionTimestamp;
public Transport(int productId, String from, String to, long timestamp) {
this.prodId = productId;
this.from = from;
this.to = to;
this.timestamp = timestamp;
}
public int getProductId() {
return prodId;
}
public String getFrom() {
return from;
}
public String getTo() {
return to;
}
public long getTimestamp() {
return timestamp;
}
@Override
public String toString() {
return "Transport{" +
"prodId=" + prodId +
", from='" + from + '\'' +
", to='" + to + '\'' +
", ingestionTimestamp='" + ingestionTimestamp + '\'' +
", timestamp='" + timestamp + '\'' +
'}';
}
public static Transport fromString(String line) {
String[] split = line.split(",");
Transport transport = new Transport(Integer.valueOf(split[0]), split[1], split[2], Long.valueOf(split[3]));
return transport;
}
}
private static Tuple2<String, Integer> getLastDestinationAndStopCountForPattern(IterativeCondition.Context<Transport> ctx, String patternName) {
return getLastDestinationAndStopCountForPattern(ctx.getEventsForPattern(patternName));
}
private static Tuple2<String, Integer> getLastDestinationAndStopCountForPattern(Iterable<Transport> events) {
Tuple2<String, Integer> locationAndStopCount = new Tuple2<>("", 0);
for (Transport transport : events) {
locationAndStopCount.f0 = transport.getTo();
locationAndStopCount.f1++;
}
return locationAndStopCount;
}
public static class MatchFunction2 extends ProcessFunction<Transport, Transport> {
@Override
public void open(Configuration config) {
}
@Override
public void processElement(Transport ride, Context context, Collector<Transport> out) throws Exception {
ride.ingestionTimestamp = context.timestamp();
out.collect(ride);
}
@Override
public void onTimer(long timestamp, OnTimerContext context, Collector<Transport> out) throws Exception {
}
}
}
3> Transport{prodId=411, from='a', to='b', ingestionTimestamp='1528384356501', timestamp='1528384357034'} Transport{prodId=411, from='b', to='c', ingestionTimestamp='1528384356502', timestamp='1528384357034'} Transport{prodId=411, from='c', to='d', ingestionTimestamp='1528384356505', timestamp='1528384357034'}
3> Transport{prodId=415, from='a', to='b', ingestionTimestamp='1528384356530', timestamp='1528384357034'} Transport{prodId=415, from='b', to='c', ingestionTimestamp='1528384356532', timestamp='1528384357034'} Transport{prodId=415, from='c', to='d', ingestionTimestamp='1528384356534', timestamp='1528384357034'}
3> Transport{prodId=419, from='a', to='b', ingestionTimestamp='1528384356549', timestamp='1528384357034'} Transport{prodId=419, from='b', to='c', ingestionTimestamp='1528384356549', timestamp='1528384357034'} Transport{prodId=419, from='c', to='d', ingestionTimestamp='1528384356554', timestamp='1528384357034'}
3> Transport{prodId=423, from='a', to='b', ingestionTimestamp='1528384356571', timestamp='1528384357034'} Transport{prodId=423, from='b', to='c', ingestionTimestamp='1528384356572', timestamp='1528384357034'} Transport{prodId=423, from='c', to='d', ingestionTimestamp='1528384356576', timestamp='1528384357034'}
3> Transport{prodId=427, from='a', to='b', ingestionTimestamp='1528384356598', timestamp='1528384357035'} Transport{prodId=427, from='b', to='c', ingestionTimestamp='1528384356600', timestamp='1528384357035'} Transport{prodId=427, from='c', to='d', ingestionTimestamp='1528384356602', timestamp='1528384357035'}
3> Transport{prodId=431, from='a', to='b', ingestionTimestamp='1528384356625', timestamp='1528384357035'} Transport{prodId=431, from='b', to='c', ingestionTimestamp='1528384356627', timestamp='1528384357035'} Transport{prodId=431, from='c', to='d', ingestionTimestamp='1528384356631', timestamp='1528384357035'}
3> Transport{prodId=435, from='a', to='b', ingestionTimestamp='1528384356656', timestamp='1528384357035'} Transport{prodId=435, from='b', to='c', ingestionTimestamp='1528384356658', timestamp='1528384357035'} Transport{prodId=435, from='c', to='d', ingestionTimestamp='1528384356660', timestamp='1528384357035'}
3> Transport{prodId=439, from='a', to='b', ingestionTimestamp='1528384356694', timestamp='1528384357035'} Transport{prodId=439, from='b', to='c', ingestionTimestamp='1528384356696', timestamp='1528384357035'} Transport{prodId=439, from='c', to='d', ingestionTimestamp='1528384356698', timestamp='1528384357035'}