Browse Source

almost working, sink still fails

master
Julio Biason 6 years ago
parent
commit
8109ca7ff2
  1. 126
      src/main/java/net/juliobiason/flink/SocketWindowWordCount.java

126
src/main/java/net/juliobiason/flink/SocketWindowWordCount.java

@ -20,10 +20,19 @@ package net.juliobiason.flink;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.java.utils.ParameterTool;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks;
import org.apache.flink.streaming.api.functions.sink.SinkFunction;
import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction.Context;
import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction;
import org.apache.flink.streaming.api.watermark.Watermark;
import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
import org.apache.flink.util.Collector;
/**
@ -57,37 +66,84 @@ public class SocketWindowWordCount {
"type the input text into the command line");
return;
}
System.out.println("Connecting to " + hostname);
System.out.println("And port " + port);
// get the execution environment
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
env.setParallelism(1);
// get input data by connecting to the socket
DataStream<String> text = env.socketTextStream(hostname, port, "\n");
// parse the data, group it, window it, and aggregate the counts
DataStream<WordWithCount> windowCounts = text
DataStream<WordEvent> mainStream = text
.flatMap(new FlatMapFunction<String, WordEvent>() {
@Override
public void flatMap(String input, Collector<WordEvent> output) {
WordEvent event = new WordEvent(input);
System.out.println("CREATE\t " + event);
output.collect(event);
}
})
.flatMap(new FlatMapFunction<String, WordWithCount>() {
.assignTimestampsAndWatermarks(new AssignerWithPeriodicWatermarks<WordEvent>() {
private long currentMaxTimestamp = 0;
@Override
public void flatMap(String value, Collector<WordWithCount> out) {
for (String word : value.split("\\s")) {
out.collect(new WordWithCount(word, 1L));
public final long extractTimestamp(WordEvent element, long previousElementTimestamp) {
long eventTimestamp = element.getTimestamp();
currentMaxTimestamp = Math.max(eventTimestamp, currentMaxTimestamp);
System.out.println("MAXTIMESTAMP\t " + currentMaxTimestamp);
return eventTimestamp;
}
@Override
public final Watermark getCurrentWatermark() {
long watermarkTime = currentMaxTimestamp - 10;
// System.out.println("WATERMARK\t " + watermarkTime);
return new Watermark(watermarkTime);
}
})
.keyBy(record -> record.getWord())
.window(TumblingEventTimeWindows.of(Time.seconds(30)))
.allowedLateness(Time.seconds(90))
.reduce(
new ReduceFunction<WordEvent>() {
public WordEvent reduce(WordEvent element1, WordEvent element2) {
long total = element1.getCount() + element2.getCount();
System.out.print("REDUCE\t " + element1 + " + " + element2 + " = " + total + "\n");
return new WordEvent(element1.getTimestamp(),
element1.getWord(),
total);
}
},
new ProcessWindowFunction<WordEvent, WordEvent, String, TimeWindow>() {
public void process(String key, Context context, Iterable<WordEvent> values, Collector<WordEvent> out) {
TimeWindow window = context.window();
for (WordEvent word: values) {
System.out.println("MOVE\t " + word + " to " + window.getStart());
out.collect(new WordEvent(window.getStart(),
word.getWord(),
word.getCount()));
}
}
}
);
.keyBy("word")
.timeWindow(Time.seconds(5))
.reduce(new ReduceFunction<WordWithCount>() {
mainStream.
.flatMap(new FlatMapFunction<WordEvent, WordEvent>() {
@Override
public WordWithCount reduce(WordWithCount a, WordWithCount b) {
return new WordWithCount(a.word, a.count + b.count);
public void flatMap(WordEvent input, Collector<WordEvent> output) {
output.collect(input);
}
});
// print the results with a single thread, rather than in parallel
windowCounts.print().setParallelism(1);
// .addSink(new SinkFunction<WordEvent>() {
// @Override
// public synchronized void invoke(
// WordEvent word,
// org.apache.flink.streaming.api.functions.sink.SinkFunction.Context ctx)
// throws Exception {
// System.out.println("SINK\t " + word);
// }
// });
env.execute("Socket Window WordCount");
}
@ -95,23 +151,43 @@ public class SocketWindowWordCount {
// ------------------------------------------------------------------------
/**
* Data type for words with count.
* The event of a word.
*/
public static class WordWithCount {
public String word;
public long count;
public WordWithCount() {}
public static class WordEvent {
private long timestamp;
private String word;
private long count;
public WordEvent(String input) {
String[] frags = input.split(" ", 2);
this.timestamp = Integer.parseInt(frags[0]) * 1000; // must be millis
this.word = frags[1];
this.count = 1;
}
public WordWithCount(String word, long count) {
public WordEvent(long timestamp, String word, long count) {
this.timestamp = timestamp;
this.word = word;
this.count = count;
}
public long getTimestamp() {
return this.timestamp;
}
public String getWord() {
return this.word;
}
public long getCount() {
return this.count;
}
@Override
public String toString() {
return word + " : " + count;
return String.format("%s at %d",
this.word,
(this.timestamp / 1000)); // still display as secs
}
}
}

Loading…
Cancel
Save