|
19 | 19 |
|
20 | 20 | import static org.hamcrest.Matchers.greaterThanOrEqualTo;
|
21 | 21 | import static org.hamcrest.Matchers.hasItem;
|
| 22 | +import static org.hamcrest.Matchers.hasItems; |
22 | 23 | import static org.hamcrest.Matchers.not;
|
23 | 24 | import static org.junit.Assert.assertEquals;
|
24 | 25 | import static org.junit.Assert.assertFalse;
|
|
36 | 37 | import org.apache.beam.sdk.Pipeline;
|
37 | 38 | import org.apache.beam.sdk.coders.BigEndianIntegerCoder;
|
38 | 39 | import org.apache.beam.sdk.coders.Coder;
|
| 40 | +import org.apache.beam.sdk.coders.InstantCoder; |
39 | 41 | import org.apache.beam.sdk.coders.SerializableCoder;
|
40 | 42 | import org.apache.beam.sdk.testing.TestPipeline;
|
41 | 43 | import org.apache.beam.sdk.transforms.Create;
|
@@ -334,6 +336,9 @@ List<OutputT> takeOutputElements() {
|
334 | 336 | return tester.takeOutputElements();
|
335 | 337 | }
|
336 | 338 |
|
| 339 | + public Instant getWatermarkHold() { |
| 340 | + return stateInternals.getWatermarkHold(); |
| 341 | + } |
337 | 342 | }
|
338 | 343 |
|
339 | 344 | private static class OutputWindowedValueToDoFnTester<OutputT>
|
@@ -429,6 +434,53 @@ public void testTrivialProcessFnPropagatesOutputsWindowsAndTimestamp() throws Ex
|
429 | 434 | }
|
430 | 435 | }
|
431 | 436 |
|
| 437 | + private static class WatermarkUpdateFn extends DoFn<Instant, String> { |
| 438 | + @ProcessElement |
| 439 | + public void process(ProcessContext c, OffsetRangeTracker tracker) { |
| 440 | + for (long i = tracker.currentRestriction().getFrom(); tracker.tryClaim(i); ++i) { |
| 441 | + c.updateWatermark(c.element().plus(Duration.standardSeconds(i))); |
| 442 | + c.output(String.valueOf(i)); |
| 443 | + } |
| 444 | + } |
| 445 | + |
| 446 | + @GetInitialRestriction |
| 447 | + public OffsetRange getInitialRestriction(Instant elem) { |
| 448 | + throw new IllegalStateException("Expected to be supplied explicitly in this test"); |
| 449 | + } |
| 450 | + |
| 451 | + @NewTracker |
| 452 | + public OffsetRangeTracker newTracker(OffsetRange range) { |
| 453 | + return new OffsetRangeTracker(range); |
| 454 | + } |
| 455 | + } |
| 456 | + |
| 457 | + @Test |
| 458 | + public void testUpdatesWatermark() throws Exception { |
| 459 | + DoFn<Instant, String> fn = new WatermarkUpdateFn(); |
| 460 | + Instant base = Instant.now(); |
| 461 | + |
| 462 | + ProcessFnTester<Instant, String, OffsetRange, OffsetRangeTracker> tester = |
| 463 | + new ProcessFnTester<>( |
| 464 | + base, |
| 465 | + fn, |
| 466 | + InstantCoder.of(), |
| 467 | + SerializableCoder.of(OffsetRange.class), |
| 468 | + 3, |
| 469 | + MAX_BUNDLE_DURATION); |
| 470 | + |
| 471 | + tester.startElement(base, new OffsetRange(0, 8)); |
| 472 | + assertThat(tester.takeOutputElements(), hasItems("0", "1", "2")); |
| 473 | + assertEquals(base.plus(Duration.standardSeconds(2)), tester.getWatermarkHold()); |
| 474 | + |
| 475 | + assertTrue(tester.advanceProcessingTimeBy(Duration.standardSeconds(1))); |
| 476 | + assertThat(tester.takeOutputElements(), hasItems("3", "4", "5")); |
| 477 | + assertEquals(base.plus(Duration.standardSeconds(5)), tester.getWatermarkHold()); |
| 478 | + |
| 479 | + assertTrue(tester.advanceProcessingTimeBy(Duration.standardSeconds(1))); |
| 480 | + assertThat(tester.takeOutputElements(), hasItems("6", "7")); |
| 481 | + assertEquals(null, tester.getWatermarkHold()); |
| 482 | + } |
| 483 | + |
432 | 484 | /**
|
433 | 485 | * A splittable {@link DoFn} that generates the sequence [init, init + total).
|
434 | 486 | */
|
|
0 commit comments