From d24d936d9a88b74b96abaec8bad203957a6f6345 Mon Sep 17 00:00:00 2001 From: Raphael Feng Date: Wed, 7 Oct 2015 17:06:43 -0700 Subject: [PATCH 1/9] Fix a typo in the comment of KinesisClientLibConfiguration Change "this msy be set to a higher number" to "this may be set to a higher number" --- .../clientlibrary/lib/worker/KinesisClientLibConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java index 9829c13e..26ceac8d 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java @@ -40,7 +40,7 @@ public class KinesisClientLibConfiguration { /** * Fail over time in milliseconds. A worker which does not renew it's lease within this time interval * will be regarded as having problems and it's shards will be assigned to other workers. - * For applications that have a large number of shards, this msy be set to a higher number to reduce + * For applications that have a large number of shards, this may be set to a higher number to reduce * the number of DynamoDB IOPS required for tracking leases. */ public static final long DEFAULT_FAILOVER_TIME_MILLIS = 10000L; From 9fbc9a34d0d0ba0ca96584a54d2fb1477c5e11e5 Mon Sep 17 00:00:00 2001 From: Andrew Shore Date: Mon, 8 Aug 2016 13:12:32 -0700 Subject: [PATCH 2/9] Removing explicit dependency on core --- pom.xml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pom.xml b/pom.xml index f6648d81..2a3b86ee 100644 --- a/pom.xml +++ b/pom.xml @@ -32,11 +32,6 @@ - - com.amazonaws - aws-java-sdk-core - ${aws-java-sdk.version} - com.amazonaws aws-java-sdk-dynamodb From 38e2adb38978443cc149de13966092ae1a659f07 Mon Sep 17 00:00:00 2001 From: "Pfifer, Justin" Date: Thu, 11 Aug 2016 08:41:39 -0700 Subject: [PATCH 3/9] Move to snapshot for next release which will be 1.7.0 --- META-INF/MANIFEST.MF | 2 +- README.md | 3 +++ pom.xml | 2 +- .../lib/worker/KinesisClientLibConfiguration.java | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/META-INF/MANIFEST.MF b/META-INF/MANIFEST.MF index 6cdb516c..f6675044 100644 --- a/META-INF/MANIFEST.MF +++ b/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: Amazon Kinesis Client Library for Java Bundle-SymbolicName: com.amazonaws.kinesisclientlibrary;singleton:=true -Bundle-Version: 1.6.5 +Bundle-Version: 1.7.0 Bundle-Vendor: Amazon Technologies, Inc Bundle-RequiredExecutionEnvironment: JavaSE-1.7 Require-Bundle: org.apache.commons.codec;bundle-version="1.6", diff --git a/README.md b/README.md index df790156..cdc7dd7d 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,9 @@ For producer-side developers using the **[Kinesis Producer Library (KPL)][kinesi To make it easier for developers to write record processors in other languages, we have implemented a Java based daemon, called MultiLangDaemon that does all the heavy lifting. Our approach has the daemon spawn a sub-process, which in turn runs the record processor, which can be written in any language. The MultiLangDaemon process and the record processor sub-process communicate with each other over [STDIN and STDOUT using a defined protocol][multi-lang-protocol]. There will be a one to one correspondence amongst record processors, child processes, and shards. For Python developers specifically, we have abstracted these implementation details away and [expose an interface][kclpy] that enables you to focus on writing record processing logic in Python. This approach enables KCL to be language agnostic, while providing identical features and similar parallel processing model across all languages. ## Release Notes +### Release 1.7.0 (???) +* Place holder for release 1.7.0 + ### Release 1.6.5 (July 25, 2016) * Change LeaseManager to call DescribeTable before attempting to create the lease table. * [Issue #36](https://github.com/awslabs/amazon-kinesis-client/issues/36) diff --git a/pom.xml b/pom.xml index f6648d81..7b740102 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ amazon-kinesis-client jar Amazon Kinesis Client Library for Java - 1.6.5 + 1.7.0-SNAPSHOT The Amazon Kinesis Client Library for Java enables Java developers to easily consume and process data from Amazon Kinesis. diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java index 8e07e501..7f2adc9a 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java @@ -119,7 +119,7 @@ public class KinesisClientLibConfiguration { /** * User agent set when Amazon Kinesis Client Library makes AWS requests. */ - public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.6.5"; + public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.7.0"; /** * KCL will validate client provided sequence numbers with a call to Amazon Kinesis before checkpointing for calls From aa47fce30bec2dab91a1e3c9a2148fad06b3f33e Mon Sep 17 00:00:00 2001 From: "Pfifer, Justin" Date: Fri, 5 Aug 2016 09:28:07 -0700 Subject: [PATCH 4/9] Added Time Based Iterator Support Added support for time based iterators. Time based iterators are only used if there is no current checkpoint for that shard, otherwise the sequence number of the checkpoint is used. --- NOTICE.txt | 2 +- pom.xml | 2 +- .../lib/checkpoint/SentinelCheckpoint.java | 8 +- .../lib/worker/InitialPositionInStream.java | 12 +- .../InitialPositionInStreamExtended.java | 78 ++++ .../lib/worker/InitializeTask.java | 9 +- .../worker/KinesisClientLibConfiguration.java | 39 +- .../lib/worker/KinesisDataFetcher.java | 67 +++- .../clientlibrary/lib/worker/ProcessTask.java | 6 +- .../lib/worker/ShardConsumer.java | 5 +- .../lib/worker/ShardSyncTask.java | 12 +- .../lib/worker/ShardSyncTaskManager.java | 6 +- .../clientlibrary/lib/worker/ShardSyncer.java | 72 +++- .../lib/worker/ShutdownTask.java | 6 +- .../lib/worker/StreamConfig.java | 9 +- .../clientlibrary/lib/worker/Worker.java | 18 +- .../clientlibrary/proxies/IKinesisProxy.java | 39 +- .../clientlibrary/proxies/KinesisProxy.java | 42 ++- ...etricsCollectingKinesisProxyDecorator.java | 37 +- .../types/ExtendedSequenceNumber.java | 23 +- .../worker/BlockOnParentShardTaskTest.java | 1 - .../KinesisClientLibConfigurationTest.java | 58 ++- .../lib/worker/KinesisDataFetcherTest.java | 69 ++-- .../lib/worker/ProcessTaskTest.java | 6 +- .../worker/SequenceNumberValidatorTest.java | 6 +- .../lib/worker/ShardConsumerTest.java | 110 +++++- .../worker/ShardSyncTaskIntegrationTest.java | 7 +- .../lib/worker/ShardSyncerTest.java | 336 +++++++++++++++--- .../lib/worker/ShutdownTaskTest.java | 41 +-- .../clientlibrary/lib/worker/WorkerTest.java | 57 ++- .../proxies/KinesisLocalFileProxy.java | 88 ++++- .../util/KinesisLocalFileDataCreator.java | 19 +- 32 files changed, 1041 insertions(+), 249 deletions(-) create mode 100644 src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/InitialPositionInStreamExtended.java diff --git a/NOTICE.txt b/NOTICE.txt index 79e11d38..650c34d7 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1,3 +1,3 @@ AmazonKinesisClientLibrary -Copyright 2012-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/pom.xml b/pom.xml index f6648d81..b2da769e 100644 --- a/pom.xml +++ b/pom.xml @@ -93,7 +93,7 @@ com.amazonaws DynamoDBLocal - 1.10.5.1 + 1.11.0.1 test diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/SentinelCheckpoint.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/SentinelCheckpoint.java index 65e00d22..d4442b82 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/SentinelCheckpoint.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/SentinelCheckpoint.java @@ -1,5 +1,5 @@ /* - * Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Amazon Software License (the "License"). * You may not use this file except in compliance with the License. @@ -31,5 +31,9 @@ public enum SentinelCheckpoint { /** * We've completely processed all records in this shard. */ - SHARD_END; + SHARD_END, + /** + * Start from the record at or after the specified server-side timestamp. + */ + AT_TIMESTAMP } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/InitialPositionInStream.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/InitialPositionInStream.java index 241683b1..94f9b455 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/InitialPositionInStream.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/InitialPositionInStream.java @@ -1,5 +1,5 @@ /* - * Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Amazon Software License (the "License"). * You may not use this file except in compliance with the License. @@ -19,14 +19,18 @@ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; * This is used during initial application bootstrap (when a checkpoint doesn't exist for a shard or its parents). */ public enum InitialPositionInStream { - /** * Start after the most recent data record (fetch new data). */ LATEST, - + /** * Start from the oldest available data record. */ - TRIM_HORIZON; + TRIM_HORIZON, + + /** + * Start from the record at or after the specified server-side timestamp. + */ + AT_TIMESTAMP } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/InitialPositionInStreamExtended.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/InitialPositionInStreamExtended.java new file mode 100644 index 00000000..6a9948c7 --- /dev/null +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/InitialPositionInStreamExtended.java @@ -0,0 +1,78 @@ +/* + * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package com.amazonaws.services.kinesis.clientlibrary.lib.worker; + +import java.util.Date; + +/** + * Class that houses the entities needed to specify the position in the stream from where a new application should + * start. + */ +class InitialPositionInStreamExtended { + + private final InitialPositionInStream position; + private final Date timestamp; + + /** + * This is scoped as private to forbid callers from using it directly and to convey the intent to use the + * static methods instead. + * + * @param position One of LATEST, TRIM_HORIZON, or AT_TIMESTAMP. The Amazon Kinesis Client Library will start + * fetching records from this position when the application starts up if there are no checkpoints. + * If there are checkpoints, we will process records from the checkpoint position. + * @param timestamp The timestamp to use with the AT_TIMESTAMP value for initialPositionInStream. + */ + private InitialPositionInStreamExtended(final InitialPositionInStream position, final Date timestamp) { + this.position = position; + this.timestamp = timestamp; + } + + /** + * Get the initial position in the stream where the application should start from. + * + * @return The initial position in stream. + */ + protected InitialPositionInStream getInitialPositionInStream() { + return this.position; + } + + /** + * Get the timestamp from where we need to start the application. + * Valid only for initial position of type AT_TIMESTAMP, returns null for other positions. + * + * @return The timestamp from where we need to start the application. + */ + protected Date getTimestamp() { + return this.timestamp; + } + + protected static InitialPositionInStreamExtended newInitialPosition(final InitialPositionInStream position) { + switch (position) { + case LATEST: + return new InitialPositionInStreamExtended(InitialPositionInStream.LATEST, null); + case TRIM_HORIZON: + return new InitialPositionInStreamExtended(InitialPositionInStream.TRIM_HORIZON, null); + default: + throw new IllegalArgumentException("Invalid InitialPosition: " + position); + } + } + + protected static InitialPositionInStreamExtended newInitialPositionAtTimestamp(final Date timestamp) { + if (timestamp == null) { + throw new IllegalArgumentException("Timestamp must be specified for InitialPosition AT_TIMESTAMP"); + } + return new InitialPositionInStreamExtended(InitialPositionInStream.AT_TIMESTAMP, timestamp); + } +} diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/InitializeTask.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/InitializeTask.java index 617813a4..262b98c7 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/InitializeTask.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/InitializeTask.java @@ -1,5 +1,5 @@ /* - * Copyright 2012-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Amazon Software License (the "License"). * You may not use this file except in compliance with the License. @@ -41,6 +41,7 @@ class InitializeTask implements ITask { private final RecordProcessorCheckpointer recordProcessorCheckpointer; // Back off for this interval if we encounter a problem (exception) private final long backoffTimeMillis; + private final StreamConfig streamConfig; /** * Constructor. @@ -50,13 +51,15 @@ class InitializeTask implements ITask { ICheckpoint checkpoint, RecordProcessorCheckpointer recordProcessorCheckpointer, KinesisDataFetcher dataFetcher, - long backoffTimeMillis) { + long backoffTimeMillis, + StreamConfig streamConfig) { this.shardInfo = shardInfo; this.recordProcessor = recordProcessor; this.checkpoint = checkpoint; this.recordProcessorCheckpointer = recordProcessorCheckpointer; this.dataFetcher = dataFetcher; this.backoffTimeMillis = backoffTimeMillis; + this.streamConfig = streamConfig; } /* @@ -74,7 +77,7 @@ class InitializeTask implements ITask { LOG.debug("Initializing ShardId " + shardInfo.getShardId()); ExtendedSequenceNumber initialCheckpoint = checkpoint.getCheckpoint(shardInfo.getShardId()); - dataFetcher.initialize(initialCheckpoint.getSequenceNumber()); + dataFetcher.initialize(initialCheckpoint.getSequenceNumber(), streamConfig.getInitialPositionInStream()); recordProcessorCheckpointer.setLargestPermittedCheckpointValue(initialCheckpoint); recordProcessorCheckpointer.setInitialCheckpointValue(initialCheckpoint); diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java index 8e07e501..0d45d359 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java @@ -14,6 +14,7 @@ */ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; +import java.util.Date; import java.util.Set; import com.amazonaws.ClientConfiguration; @@ -185,6 +186,7 @@ public class KinesisClientLibConfiguration { private int maxLeasesToStealAtOneTime; private int initialLeaseTableReadCapacity; private int initialLeaseTableWriteCapacity; + private InitialPositionInStreamExtended initialPositionInStreamExtended; /** * Constructor. @@ -263,7 +265,6 @@ public class KinesisClientLibConfiguration { * with a call to Amazon Kinesis before checkpointing for calls to * {@link RecordProcessorCheckpointer#checkpoint(String)} * @param regionName The region name for the service - * */ // CHECKSTYLE:IGNORE HiddenFieldCheck FOR NEXT 26 LINES // CHECKSTYLE:IGNORE ParameterNumber FOR NEXT 26 LINES @@ -330,6 +331,8 @@ public class KinesisClientLibConfiguration { this.maxLeasesToStealAtOneTime = DEFAULT_MAX_LEASES_TO_STEAL_AT_ONE_TIME; this.initialLeaseTableReadCapacity = DEFAULT_INITIAL_LEASE_TABLE_READ_CAPACITY; this.initialLeaseTableWriteCapacity = DEFAULT_INITIAL_LEASE_TABLE_WRITE_CAPACITY; + this.initialPositionInStreamExtended = + InitialPositionInStreamExtended.newInitialPosition(initialPositionInStream); } // Check if value is positive, otherwise throw an exception @@ -580,6 +583,22 @@ public class KinesisClientLibConfiguration { return initialLeaseTableWriteCapacity; } + /** + * Keeping it protected to forbid outside callers from depending on this internal object. + * @return The initialPositionInStreamExtended object. + */ + protected InitialPositionInStreamExtended getInitialPositionInStreamExtended() { + return initialPositionInStreamExtended; + } + + /** + * @return The timestamp from where we need to start the application. + * Valid only for initial position of type AT_TIMESTAMP, returns null for other positions. + */ + public Date getTimestampAtInitialPositionInStream() { + return initialPositionInStreamExtended.getTimestamp(); + } + // CHECKSTYLE:IGNORE HiddenFieldCheck FOR NEXT 190 LINES /** * @param tableName name of the lease table in DynamoDB @@ -600,13 +619,25 @@ public class KinesisClientLibConfiguration { } /** - * @param initialPositionInStream One of LATEST or TRIM_HORIZON. The Amazon Kinesis Client Library will start - * fetching records from this position when the application starts up if there are no checkpoints. If there - * are checkpoints, we will process records from the checkpoint position. + * @param initialPositionInStream One of LATEST or TRIM_HORIZON. The Amazon Kinesis Client Library + * will start fetching records from this position when the application starts up if there are no checkpoints. + * If there are checkpoints, we will process records from the checkpoint position. * @return KinesisClientLibConfiguration */ public KinesisClientLibConfiguration withInitialPositionInStream(InitialPositionInStream initialPositionInStream) { this.initialPositionInStream = initialPositionInStream; + this.initialPositionInStreamExtended = + InitialPositionInStreamExtended.newInitialPosition(initialPositionInStream); + return this; + } + + /** + * @param timestamp The timestamp to use with the AT_TIMESTAMP value for initialPositionInStream. + * @return KinesisClientLibConfiguration + */ + public KinesisClientLibConfiguration withTimestampAtInitialPositionInStream(Date timestamp) { + this.initialPositionInStream = InitialPositionInStream.AT_TIMESTAMP; + this.initialPositionInStreamExtended = InitialPositionInStreamExtended.newInitialPositionAtTimestamp(timestamp); return this; } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisDataFetcher.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisDataFetcher.java index 14d0448c..2ce3152a 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisDataFetcher.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisDataFetcher.java @@ -1,5 +1,5 @@ /* - * Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Amazon Software License (the "License"). * You may not use this file except in compliance with the License. @@ -25,6 +25,8 @@ import com.amazonaws.services.kinesis.clientlibrary.proxies.IKinesisProxy; import com.amazonaws.services.kinesis.clientlibrary.proxies.MetricsCollectingKinesisProxyDecorator; import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber; +import java.util.Date; + /** * Used to get data from Amazon Kinesis. Tracks iterator state internally. */ @@ -41,8 +43,7 @@ class KinesisDataFetcher { /** * * @param kinesisProxy Kinesis proxy - * @param shardId shardId (we'll fetch data for this shard) - * @param checkpoint used to get current checkpoint from which to start fetching records + * @param shardInfo The shardInfo object. */ public KinesisDataFetcher(IKinesisProxy kinesisProxy, ShardInfo shardInfo) { this.shardId = shardInfo.getShardId(); @@ -83,17 +84,18 @@ class KinesisDataFetcher { /** * Initializes this KinesisDataFetcher's iterator based on the checkpointed sequence number. * @param initialCheckpoint Current checkpoint sequence number for this shard. - * + * @param initialPositionInStream The initialPositionInStream. */ - public void initialize(String initialCheckpoint) { + public void initialize(String initialCheckpoint, InitialPositionInStreamExtended initialPositionInStream) { LOG.info("Initializing shard " + shardId + " with " + initialCheckpoint); - advanceIteratorTo(initialCheckpoint); + advanceIteratorTo(initialCheckpoint, initialPositionInStream); isInitialized = true; } - - public void initialize(ExtendedSequenceNumber initialCheckpoint) { + + public void initialize(ExtendedSequenceNumber initialCheckpoint, + InitialPositionInStreamExtended initialPositionInStream) { LOG.info("Initializing shard " + shardId + " with " + initialCheckpoint.getSequenceNumber()); - advanceIteratorTo(initialCheckpoint.getSequenceNumber()); + advanceIteratorTo(initialCheckpoint.getSequenceNumber(), initialPositionInStream); isInitialized = true; } @@ -101,14 +103,17 @@ class KinesisDataFetcher { * Advances this KinesisDataFetcher's internal iterator to be at the passed-in sequence number. * * @param sequenceNumber advance the iterator to the record at this sequence number. + * @param initialPositionInStream The initialPositionInStream. */ - void advanceIteratorTo(String sequenceNumber) { + void advanceIteratorTo(String sequenceNumber, InitialPositionInStreamExtended initialPositionInStream) { if (sequenceNumber == null) { throw new IllegalArgumentException("SequenceNumber should not be null: shardId " + shardId); } else if (sequenceNumber.equals(SentinelCheckpoint.LATEST.toString())) { - nextIterator = getIterator(ShardIteratorType.LATEST.toString(), null); + nextIterator = getIterator(ShardIteratorType.LATEST.toString()); } else if (sequenceNumber.equals(SentinelCheckpoint.TRIM_HORIZON.toString())) { - nextIterator = getIterator(ShardIteratorType.TRIM_HORIZON.toString(), null); + nextIterator = getIterator(ShardIteratorType.TRIM_HORIZON.toString()); + } else if (sequenceNumber.equals(SentinelCheckpoint.AT_TIMESTAMP.toString())) { + nextIterator = getIterator(initialPositionInStream.getTimestamp()); } else if (sequenceNumber.equals(SentinelCheckpoint.SHARD_END.toString())) { nextIterator = null; } else { @@ -120,8 +125,8 @@ class KinesisDataFetcher { } /** - * @param iteratorType - * @param sequenceNumber + * @param iteratorType The iteratorType - either AT_SEQUENCE_NUMBER or AFTER_SEQUENCE_NUMBER. + * @param sequenceNumber The sequenceNumber. * * @return iterator or null if we catch a ResourceNotFound exception */ @@ -139,6 +144,40 @@ class KinesisDataFetcher { return iterator; } + /** + * @param iteratorType The iteratorType - either TRIM_HORIZON or LATEST. + * @return iterator or null if we catch a ResourceNotFound exception + */ + private String getIterator(String iteratorType) { + String iterator = null; + try { + if (LOG.isDebugEnabled()) { + LOG.debug("Calling getIterator for " + shardId + " and iterator type " + iteratorType); + } + iterator = kinesisProxy.getIterator(shardId, iteratorType); + } catch (ResourceNotFoundException e) { + LOG.info("Caught ResourceNotFoundException when getting an iterator for shard " + shardId, e); + } + return iterator; + } + + /** + * @param timestamp The timestamp. + * @return iterator or null if we catch a ResourceNotFound exception + */ + private String getIterator(Date timestamp) { + String iterator = null; + try { + if (LOG.isDebugEnabled()) { + LOG.debug("Calling getIterator for " + shardId + " and timestamp " + timestamp); + } + iterator = kinesisProxy.getIterator(shardId, timestamp); + } catch (ResourceNotFoundException e) { + LOG.info("Caught ResourceNotFoundException when getting an iterator for shard " + shardId, e); + } + return iterator; + } + /** * @return the shardEndReached */ diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ProcessTask.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ProcessTask.java index 47ee7a5d..db0970d5 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ProcessTask.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ProcessTask.java @@ -1,5 +1,5 @@ /* - * Copyright 2012-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Amazon Software License (the "License"). * You may not use this file except in compliance with the License. @@ -259,8 +259,8 @@ class ProcessTask implements ITask { * Advance the iterator to after the greatest processed sequence number (remembered by * recordProcessorCheckpointer). */ - dataFetcher.advanceIteratorTo( - recordProcessorCheckpointer.getLargestPermittedCheckpointValue().getSequenceNumber()); + dataFetcher.advanceIteratorTo(recordProcessorCheckpointer.getLargestPermittedCheckpointValue() + .getSequenceNumber(), streamConfig.getInitialPositionInStream()); // Try a second time - if we fail this time, expose the failure. try { diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumer.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumer.java index 8445e684..10dacc04 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumer.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumer.java @@ -1,5 +1,5 @@ /* - * Copyright 2012-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Amazon Software License (the "License"). * You may not use this file except in compliance with the License. @@ -252,7 +252,8 @@ class ShardConsumer { checkpoint, recordProcessorCheckpointer, dataFetcher, - taskBackoffTimeMillis); + taskBackoffTimeMillis, + streamConfig); break; case PROCESSING: nextTask = diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTask.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTask.java index f0db8cda..ddfb8459 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTask.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTask.java @@ -1,5 +1,5 @@ /* - * Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Amazon Software License (the "License"). * You may not use this file except in compliance with the License. @@ -33,7 +33,7 @@ class ShardSyncTask implements ITask { private final IKinesisProxy kinesisProxy; private final ILeaseManager leaseManager; - private InitialPositionInStream initialPosition; + private InitialPositionInStreamExtended initialPosition; private final boolean cleanupLeasesUponShardCompletion; private final long shardSyncTaskIdleTimeMillis; private final TaskType taskType = TaskType.SHARDSYNC; @@ -41,13 +41,13 @@ class ShardSyncTask implements ITask { /** * @param kinesisProxy Used to fetch information about the stream (e.g. shard list) * @param leaseManager Used to fetch and create leases - * @param initialPosition One of LATEST or TRIM_HORIZON. Amazon Kinesis Client Library will start processing records - * from this point in the stream (when an application starts up for the first time) except for shards that - * already have a checkpoint (and their descendant shards). + * @param initialPositionInStream One of LATEST, TRIM_HORIZON or AT_TIMESTAMP. Amazon Kinesis Client Library will + * start processing records from this point in the stream (when an application starts up for the first time) + * except for shards that already have a checkpoint (and their descendant shards). */ ShardSyncTask(IKinesisProxy kinesisProxy, ILeaseManager leaseManager, - InitialPositionInStream initialPositionInStream, + InitialPositionInStreamExtended initialPositionInStream, boolean cleanupLeasesUponShardCompletion, long shardSyncTaskIdleTimeMillis) { this.kinesisProxy = kinesisProxy; diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTaskManager.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTaskManager.java index 7fffe123..c1bfae76 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTaskManager.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTaskManager.java @@ -1,5 +1,5 @@ /* - * Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Amazon Software License (the "License"). * You may not use this file except in compliance with the License. @@ -42,7 +42,7 @@ class ShardSyncTaskManager { private final ILeaseManager leaseManager; private final IMetricsFactory metricsFactory; private final ExecutorService executorService; - private final InitialPositionInStream initialPositionInStream; + private final InitialPositionInStreamExtended initialPositionInStream; private boolean cleanupLeasesUponShardCompletion; private final long shardSyncIdleTimeMillis; @@ -61,7 +61,7 @@ class ShardSyncTaskManager { */ ShardSyncTaskManager(final IKinesisProxy kinesisProxy, final ILeaseManager leaseManager, - final InitialPositionInStream initialPositionInStream, + final InitialPositionInStreamExtended initialPositionInStream, final boolean cleanupLeasesUponShardCompletion, final long shardSyncIdleTimeMillis, final IMetricsFactory metricsFactory, diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncer.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncer.java index 97298d48..52944200 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncer.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncer.java @@ -1,5 +1,5 @@ /* - * Copyright 2012-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Amazon Software License (the "License"). * You may not use this file except in compliance with the License. @@ -59,7 +59,7 @@ class ShardSyncer { static synchronized void bootstrapShardLeases(IKinesisProxy kinesisProxy, ILeaseManager leaseManager, - InitialPositionInStream initialPositionInStream, + InitialPositionInStreamExtended initialPositionInStream, boolean cleanupLeasesOfCompletedShards) throws DependencyException, InvalidStateException, ProvisionedThroughputException, KinesisClientLibIOException { syncShardLeases(kinesisProxy, leaseManager, initialPositionInStream, cleanupLeasesOfCompletedShards); @@ -82,7 +82,7 @@ class ShardSyncer { */ static synchronized void checkAndCreateLeasesForNewShards(IKinesisProxy kinesisProxy, ILeaseManager leaseManager, - InitialPositionInStream initialPositionInStream, + InitialPositionInStreamExtended initialPositionInStream, boolean cleanupLeasesOfCompletedShards) throws DependencyException, InvalidStateException, ProvisionedThroughputException, KinesisClientLibIOException { syncShardLeases(kinesisProxy, leaseManager, initialPositionInStream, cleanupLeasesOfCompletedShards); @@ -106,7 +106,7 @@ class ShardSyncer { // CHECKSTYLE:OFF CyclomaticComplexity private static synchronized void syncShardLeases(IKinesisProxy kinesisProxy, ILeaseManager leaseManager, - InitialPositionInStream initialPosition, + InitialPositionInStreamExtended initialPosition, boolean cleanupLeasesOfCompletedShards) throws DependencyException, InvalidStateException, ProvisionedThroughputException, KinesisClientLibIOException { List shards = getShardList(kinesisProxy); @@ -327,15 +327,15 @@ class ShardSyncer { * when persisting the leases in DynamoDB will ensure that we recover gracefully if we fail * before creating all the leases. * - * @param shardIds Set of all shardIds in Kinesis (we'll create new leases based on this set) + * @param shards List of all shards in Kinesis (we'll create new leases based on this set) * @param currentLeases List of current leases - * @param initialPosition One of LATEST or TRIM_HORIZON. We'll start fetching records from that location in the - * shard (when an application starts up for the first time - and there are no checkpoints). + * @param initialPosition One of LATEST, TRIM_HORIZON, or AT_TIMESTAMP. We'll start fetching records from that + * location in the shard (when an application starts up for the first time - and there are no checkpoints). * @return List of new leases to create sorted by starting sequenceNumber of the corresponding shard */ static List determineNewLeasesToCreate(List shards, List currentLeases, - InitialPositionInStream initialPosition) { + InitialPositionInStreamExtended initialPosition) { Map shardIdToNewLeaseMap = new HashMap(); Map shardIdToShardMapOfAllKinesisShards = constructShardIdToShardMap(shards); @@ -364,7 +364,32 @@ class ShardSyncer { shardIdToShardMapOfAllKinesisShards, shardIdToNewLeaseMap, memoizationContext); - if (isDescendant) { + + /** + * If the shard is a descendant and the specified initial position is AT_TIMESTAMP, then the + * checkpoint should be set to AT_TIMESTAMP, else to TRIM_HORIZON. For AT_TIMESTAMP, we will add a + * lease just like we do for TRIM_HORIZON. However we will only return back records with server-side + * timestamp at or after the specified initial position timestamp. + * + * Shard structure (each level depicts a stream segment): + * 0 1 2 3 4 5 - shards till epoch 102 + * \ / \ / | | + * 6 7 4 5 - shards from epoch 103 - 205 + * \ / | /\ + * 8 4 9 10 - shards from epoch 206 (open - no ending sequenceNumber) + * + * Current leases: empty set + * + * For the above example, suppose the initial position in stream is set to AT_TIMESTAMP with + * timestamp value 206. We will then create new leases for all the shards (with checkpoint set to + * AT_TIMESTAMP), including the ancestor shards with epoch less than 206. However as we begin + * processing the ancestor shards, their checkpoints would be updated to SHARD_END and their leases + * would then be deleted since they won't have records with server-side timestamp at/after 206. And + * after that we will begin processing the descendant shards with epoch at/after 206 and we will + * return the records that meet the timestamp requirement for these shards. + */ + if (isDescendant && !initialPosition.getInitialPositionInStream() + .equals(InitialPositionInStream.AT_TIMESTAMP)) { newLease.setCheckpoint(ExtendedSequenceNumber.TRIM_HORIZON); } else { newLease.setCheckpoint(convertToCheckpoint(initialPosition)); @@ -388,8 +413,10 @@ class ShardSyncer { * Create leases for the ancestors of this shard as required. * See javadoc of determineNewLeasesToCreate() for rules and example. * - * @param shardIds Ancestors of these shards will be considered for addition into the new lease map - * @param shardIdsOfCurrentLeases + * @param shardId The shardId to check. + * @param initialPosition One of LATEST, TRIM_HORIZON, or AT_TIMESTAMP. We'll start fetching records from that + * location in the shard (when an application starts up for the first time - and there are no checkpoints). + * @param shardIdsOfCurrentLeases The shardIds for the current leases. * @param shardIdToShardMapOfAllKinesisShards ShardId->Shard map containing all shards obtained via DescribeStream. * @param shardIdToLeaseMapOfNewShards Add lease POJOs corresponding to ancestors to this map. * @param memoizationContext Memoization of shards that have been evaluated as part of the evaluation @@ -397,7 +424,7 @@ class ShardSyncer { */ // CHECKSTYLE:OFF CyclomaticComplexity static boolean checkIfDescendantAndAddNewLeasesForAncestors(String shardId, - InitialPositionInStream initialPosition, + InitialPositionInStreamExtended initialPosition, Set shardIdsOfCurrentLeases, Map shardIdToShardMapOfAllKinesisShards, Map shardIdToLeaseMapOfNewShards, @@ -449,7 +476,9 @@ class ShardSyncer { shardIdToLeaseMapOfNewShards.put(parentShardId, lease); } - if (descendantParentShardIds.contains(parentShardId)) { + if (descendantParentShardIds.contains(parentShardId) + && !initialPosition.getInitialPositionInStream() + .equals(InitialPositionInStream.AT_TIMESTAMP)) { lease.setCheckpoint(ExtendedSequenceNumber.TRIM_HORIZON); } else { lease.setCheckpoint(convertToCheckpoint(initialPosition)); @@ -457,8 +486,13 @@ class ShardSyncer { } } } else { - // This shard should be included, if the customer wants to process all records in the stream. - if (initialPosition.equals(InitialPositionInStream.TRIM_HORIZON)) { + // This shard should be included, if the customer wants to process all records in the stream or + // if the initial position is AT_TIMESTAMP. For AT_TIMESTAMP, we will add a lease just like we do + // for TRIM_HORIZON. However we will only return back records with server-side timestamp at or + // after the specified initial position timestamp. + if (initialPosition.getInitialPositionInStream().equals(InitialPositionInStream.TRIM_HORIZON) + || initialPosition.getInitialPositionInStream() + .equals(InitialPositionInStream.AT_TIMESTAMP)) { isDescendant = true; } } @@ -737,13 +771,15 @@ class ShardSyncer { return openShards; } - private static ExtendedSequenceNumber convertToCheckpoint(InitialPositionInStream position) { + private static ExtendedSequenceNumber convertToCheckpoint(InitialPositionInStreamExtended position) { ExtendedSequenceNumber checkpoint = null; - if (position.equals(InitialPositionInStream.TRIM_HORIZON)) { + if (position.getInitialPositionInStream().equals(InitialPositionInStream.TRIM_HORIZON)) { checkpoint = ExtendedSequenceNumber.TRIM_HORIZON; - } else if (position.equals(InitialPositionInStream.LATEST)) { + } else if (position.getInitialPositionInStream().equals(InitialPositionInStream.LATEST)) { checkpoint = ExtendedSequenceNumber.LATEST; + } else if (position.getInitialPositionInStream().equals(InitialPositionInStream.AT_TIMESTAMP)) { + checkpoint = ExtendedSequenceNumber.AT_TIMESTAMP; } return checkpoint; diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTask.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTask.java index ecf4873e..3ce05203 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTask.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTask.java @@ -1,5 +1,5 @@ /* - * Copyright 2012-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Amazon Software License (the "License"). * You may not use this file except in compliance with the License. @@ -42,7 +42,7 @@ class ShutdownTask implements ITask { private final ShutdownReason reason; private final IKinesisProxy kinesisProxy; private final ILeaseManager leaseManager; - private final InitialPositionInStream initialPositionInStream; + private final InitialPositionInStreamExtended initialPositionInStream; private final boolean cleanupLeasesOfCompletedShards; private final TaskType taskType = TaskType.SHUTDOWN; private final long backoffTimeMillis; @@ -56,7 +56,7 @@ class ShutdownTask implements ITask { RecordProcessorCheckpointer recordProcessorCheckpointer, ShutdownReason reason, IKinesisProxy kinesisProxy, - InitialPositionInStream initialPositionInStream, + InitialPositionInStreamExtended initialPositionInStream, boolean cleanupLeasesOfCompletedShards, ILeaseManager leaseManager, long backoffTimeMillis) { diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/StreamConfig.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/StreamConfig.java index 2b7120fd..b5c283fb 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/StreamConfig.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/StreamConfig.java @@ -1,5 +1,5 @@ /* - * Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Amazon Software License (the "License"). * You may not use this file except in compliance with the License. @@ -25,7 +25,7 @@ class StreamConfig { private final int maxRecords; private final long idleTimeInMilliseconds; private final boolean callProcessRecordsEvenForEmptyRecordList; - private InitialPositionInStream initialPositionInStream; + private InitialPositionInStreamExtended initialPositionInStream; private final boolean validateSequenceNumberBeforeCheckpointing; /** @@ -42,7 +42,7 @@ class StreamConfig { long idleTimeInMilliseconds, boolean callProcessRecordsEvenForEmptyRecordList, boolean validateSequenceNumberBeforeCheckpointing, - InitialPositionInStream initialPositionInStream) { + InitialPositionInStreamExtended initialPositionInStream) { this.streamProxy = proxy; this.maxRecords = maxRecords; this.idleTimeInMilliseconds = idleTimeInMilliseconds; @@ -82,7 +82,7 @@ class StreamConfig { /** * @return the initialPositionInStream */ - InitialPositionInStream getInitialPositionInStream() { + InitialPositionInStreamExtended getInitialPositionInStream() { return initialPositionInStream; } @@ -92,5 +92,4 @@ class StreamConfig { boolean shouldValidateSequenceNumberBeforeCheckpointing() { return validateSequenceNumberBeforeCheckpointing; } - } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java index 6aec2346..50861c29 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java @@ -64,7 +64,7 @@ public class Worker implements Runnable { private final String applicationName; private final IRecordProcessorFactory recordProcessorFactory; private final StreamConfig streamConfig; - private final InitialPositionInStream initialPosition; + private final InitialPositionInStreamExtended initialPosition; private final ICheckpoint checkpointTracker; private final long idleTimeInMilliseconds; // Backoff time when polling to check if application has finished processing @@ -212,8 +212,8 @@ public class Worker implements Runnable { config.getMaxRecords(), config.getIdleTimeBetweenReadsInMillis(), config.shouldCallProcessRecordsEvenForEmptyRecordList(), config.shouldValidateSequenceNumberBeforeCheckpointing(), - config.getInitialPositionInStream()), - config.getInitialPositionInStream(), + config.getInitialPositionInStreamExtended()), + config.getInitialPositionInStreamExtended(), config.getParentShardPollIntervalMillis(), config.getShardSyncIntervalMillis(), config.shouldCleanupLeasesUponShardCompletion(), @@ -258,9 +258,9 @@ public class Worker implements Runnable { * @param applicationName Name of the Kinesis application * @param recordProcessorFactory Used to get record processor instances for processing data from shards * @param streamConfig Stream configuration - * @param initialPositionInStream One of LATEST or TRIM_HORIZON. The KinesisClientLibrary will start fetching data - * from this location in the stream when an application starts up for the first time and there are no - * checkpoints. If there are checkpoints, we start from the checkpoint position. + * @param initialPositionInStream One of LATEST, TRIM_HORIZON, or AT_TIMESTAMP. The KinesisClientLibrary will start + * fetching data from this location in the stream when an application starts up for the first time and + * there are no checkpoints. If there are checkpoints, we start from the checkpoint position. * @param parentShardPollIntervalMillis Wait for this long between polls to check if parent shards are done * @param shardSyncIdleTimeMillis Time between tasks to sync leases and Kinesis shards * @param cleanupLeasesUponShardCompletion Clean up shards we've finished processing (don't wait till they expire in @@ -277,7 +277,7 @@ public class Worker implements Runnable { Worker(String applicationName, IRecordProcessorFactory recordProcessorFactory, StreamConfig streamConfig, - InitialPositionInStream initialPositionInStream, + InitialPositionInStreamExtended initialPositionInStream, long parentShardPollIntervalMillis, long shardSyncIdleTimeMillis, boolean cleanupLeasesUponShardCompletion, @@ -946,8 +946,8 @@ public class Worker implements Runnable { config.getIdleTimeBetweenReadsInMillis(), config.shouldCallProcessRecordsEvenForEmptyRecordList(), config.shouldValidateSequenceNumberBeforeCheckpointing(), - config.getInitialPositionInStream()), - config.getInitialPositionInStream(), + config.getInitialPositionInStreamExtended()), + config.getInitialPositionInStreamExtended(), config.getParentShardPollIntervalMillis(), config.getShardSyncIntervalMillis(), config.shouldCleanupLeasesUponShardCompletion(), diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/IKinesisProxy.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/IKinesisProxy.java index 8dbb97fa..df7f951d 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/IKinesisProxy.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/IKinesisProxy.java @@ -1,5 +1,5 @@ /* - * Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Amazon Software License (the "License"). * You may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ package com.amazonaws.services.kinesis.clientlibrary.proxies; import java.nio.ByteBuffer; +import java.util.Date; import java.util.List; import java.util.Set; @@ -72,7 +73,16 @@ public interface IKinesisProxy { /** * Fetch a shard iterator from the specified position in the shard. - * + * This is to fetch a shard iterator for ShardIteratorType AT_SEQUENCE_NUMBER or AFTER_SEQUENCE_NUMBER which + * requires the starting sequence number. + * + * NOTE: Currently this method continues to fetch iterators for ShardIteratorTypes TRIM_HORIZON, LATEST, + * AT_SEQUENCE_NUMBER and AFTER_SEQUENCE_NUMBER. + * But this behavior will change in the next release, after which this method will only serve + * AT_SEQUENCE_NUMBER or AFTER_SEQUENCE_NUMBER ShardIteratorTypes. + * We recommend users who call this method directly to use the appropriate getIterator method based on the + * ShardIteratorType. + * * @param shardId Shard id * @param iteratorEnum one of: TRIM_HORIZON, LATEST, AT_SEQUENCE_NUMBER, AFTER_SEQUENCE_NUMBER * @param sequenceNumber the sequence number - must be null unless iteratorEnum is AT_SEQUENCE_NUMBER or @@ -84,6 +94,31 @@ public interface IKinesisProxy { String getIterator(String shardId, String iteratorEnum, String sequenceNumber) throws ResourceNotFoundException, InvalidArgumentException; + /** + * Fetch a shard iterator from the specified position in the shard. + * This is to fetch a shard iterator for ShardIteratorType LATEST or TRIM_HORIZON which doesn't require a starting + * sequence number. + * + * @param shardId Shard id + * @param iteratorEnum Either TRIM_HORIZON or LATEST. + * @return shard iterator which can be used to read data from Kinesis. + * @throws ResourceNotFoundException The Kinesis stream or shard was not found + * @throws InvalidArgumentException Invalid input parameters + */ + String getIterator(String shardId, String iteratorEnum) throws ResourceNotFoundException, InvalidArgumentException; + + /** + * Fetch a shard iterator from the specified position in the shard. + * This is to fetch a shard iterator for ShardIteratorType AT_TIMESTAMP which requires the timestamp field. + * + * @param shardId Shard id + * @param timestamp The timestamp. + * @return shard iterator which can be used to read data from Kinesis. + * @throws ResourceNotFoundException The Kinesis stream or shard was not found + * @throws InvalidArgumentException Invalid input parameters + */ + String getIterator(String shardId, Date timestamp) throws ResourceNotFoundException, InvalidArgumentException; + /** * @param sequenceNumberForOrdering (optional) used for record ordering * @param explicitHashKey optionally supplied transformation of partitionkey diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxy.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxy.java index 0cf36da7..ad929c21 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxy.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxy.java @@ -1,5 +1,5 @@ /* - * Copyright 2012-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Amazon Software License (the "License"). * You may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ package com.amazonaws.services.kinesis.clientlibrary.proxies; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Date; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -40,6 +41,7 @@ import com.amazonaws.services.kinesis.model.PutRecordRequest; import com.amazonaws.services.kinesis.model.PutRecordResult; import com.amazonaws.services.kinesis.model.ResourceNotFoundException; import com.amazonaws.services.kinesis.model.Shard; +import com.amazonaws.services.kinesis.model.ShardIteratorType; import com.amazonaws.services.kinesis.model.StreamStatus; /** @@ -263,12 +265,50 @@ public class KinesisProxy implements IKinesisProxyExtended { */ @Override public String getIterator(String shardId, String iteratorType, String sequenceNumber) { + if (!iteratorType.equals(ShardIteratorType.AT_SEQUENCE_NUMBER.toString()) || !iteratorType.equals( + ShardIteratorType.AFTER_SEQUENCE_NUMBER.toString())) { + LOG.info("This method should only be used for AT_SEQUENCE_NUMBER and AFTER_SEQUENCE_NUMBER " + + "ShardIteratorTypes. For methods to use with other ShardIteratorTypes, see IKinesisProxy.java"); + } final GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest(); getShardIteratorRequest.setRequestCredentials(credentialsProvider.getCredentials()); getShardIteratorRequest.setStreamName(streamName); getShardIteratorRequest.setShardId(shardId); getShardIteratorRequest.setShardIteratorType(iteratorType); getShardIteratorRequest.setStartingSequenceNumber(sequenceNumber); + getShardIteratorRequest.setTimestamp(null); + final GetShardIteratorResult response = client.getShardIterator(getShardIteratorRequest); + return response.getShardIterator(); + } + + /** + * {@inheritDoc} + */ + @Override + public String getIterator(String shardId, String iteratorType) { + final GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest(); + getShardIteratorRequest.setRequestCredentials(credentialsProvider.getCredentials()); + getShardIteratorRequest.setStreamName(streamName); + getShardIteratorRequest.setShardId(shardId); + getShardIteratorRequest.setShardIteratorType(iteratorType); + getShardIteratorRequest.setStartingSequenceNumber(null); + getShardIteratorRequest.setTimestamp(null); + final GetShardIteratorResult response = client.getShardIterator(getShardIteratorRequest); + return response.getShardIterator(); + } + + /** + * {@inheritDoc} + */ + @Override + public String getIterator(String shardId, Date timestamp) { + final GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest(); + getShardIteratorRequest.setRequestCredentials(credentialsProvider.getCredentials()); + getShardIteratorRequest.setStreamName(streamName); + getShardIteratorRequest.setShardId(shardId); + getShardIteratorRequest.setShardIteratorType(ShardIteratorType.AT_TIMESTAMP); + getShardIteratorRequest.setStartingSequenceNumber(null); + getShardIteratorRequest.setTimestamp(timestamp); final GetShardIteratorResult response = client.getShardIterator(getShardIteratorRequest); return response.getShardIterator(); } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/MetricsCollectingKinesisProxyDecorator.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/MetricsCollectingKinesisProxyDecorator.java index 7263351e..d27fc6a1 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/MetricsCollectingKinesisProxyDecorator.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/MetricsCollectingKinesisProxyDecorator.java @@ -1,5 +1,5 @@ /* - * Copyright 2012-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Amazon Software License (the "License"). * You may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ package com.amazonaws.services.kinesis.clientlibrary.proxies; import java.nio.ByteBuffer; +import java.util.Date; import java.util.List; import java.util.Set; @@ -128,6 +129,40 @@ public class MetricsCollectingKinesisProxyDecorator implements IKinesisProxy { } } + /** + * {@inheritDoc} + */ + @Override + public String getIterator(String shardId, String iteratorEnum) + throws ResourceNotFoundException, InvalidArgumentException { + long startTime = System.currentTimeMillis(); + boolean success = false; + try { + String response = other.getIterator(shardId, iteratorEnum); + success = true; + return response; + } finally { + MetricsHelper.addSuccessAndLatency(getIteratorMetric, startTime, success, MetricsLevel.DETAILED); + } + } + + /** + * {@inheritDoc} + */ + @Override + public String getIterator(String shardId, Date timestamp) + throws ResourceNotFoundException, InvalidArgumentException { + long startTime = System.currentTimeMillis(); + boolean success = false; + try { + String response = other.getIterator(shardId, timestamp); + success = true; + return response; + } finally { + MetricsHelper.addSuccessAndLatency(getIteratorMetric, startTime, success, MetricsLevel.DETAILED); + } + } + /** * {@inheritDoc} */ diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/types/ExtendedSequenceNumber.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/types/ExtendedSequenceNumber.java index 0202a17a..1ed7ed67 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/types/ExtendedSequenceNumber.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/types/ExtendedSequenceNumber.java @@ -1,5 +1,5 @@ /* - * Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Amazon Software License (the "License"). * You may not use this file except in compliance with the License. @@ -36,9 +36,10 @@ public class ExtendedSequenceNumber implements Comparable expectedRecords = new ArrayList(); GetRecordsResult response = new GetRecordsResult(); response.setRecords(expectedRecords); - - when(kinesis.getIterator(SHARD_ID, iteratorType, null)).thenReturn(iterator); + when(kinesis.getIterator(SHARD_ID, initialPositionInStream.getTimestamp())).thenReturn(iterator); when(kinesis.getIterator(SHARD_ID, AT_SEQUENCE_NUMBER, seqNo)).thenReturn(iterator); + when(kinesis.getIterator(SHARD_ID, iteratorType)).thenReturn(iterator); when(kinesis.get(iterator, MAX_RECORDS)).thenReturn(response); ICheckpoint checkpoint = mock(ICheckpoint.class); when(checkpoint.getCheckpoint(SHARD_ID)).thenReturn(new ExtendedSequenceNumber(seqNo)); KinesisDataFetcher fetcher = new KinesisDataFetcher(kinesis, SHARD_INFO); - - fetcher.initialize(seqNo); + fetcher.initialize(seqNo, initialPositionInStream); List actualRecords = fetcher.getRecords(MAX_RECORDS).getRecords(); Assert.assertEquals(expectedRecords, actualRecords); diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ProcessTaskTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ProcessTaskTest.java index 5385d05e..6576e47f 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ProcessTaskTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ProcessTaskTest.java @@ -66,7 +66,8 @@ public class ProcessTaskTest { private final boolean callProcessRecordsForEmptyRecordList = true; // We don't want any of these tests to run checkpoint validation private final boolean skipCheckpointValidationValue = false; - private final InitialPositionInStream initialPositionInStream = InitialPositionInStream.LATEST; + private static final InitialPositionInStreamExtended INITIAL_POSITION_LATEST = + InitialPositionInStreamExtended.newInitialPosition(InitialPositionInStream.LATEST); private @Mock KinesisDataFetcher mockDataFetcher; private @Mock IRecordProcessor mockRecordProcessor; @@ -84,7 +85,8 @@ public class ProcessTaskTest { // Set up process task final StreamConfig config = new StreamConfig(null, maxRecords, idleTimeMillis, callProcessRecordsForEmptyRecordList, - skipCheckpointValidationValue, initialPositionInStream); + skipCheckpointValidationValue, + INITIAL_POSITION_LATEST); final ShardInfo shardInfo = new ShardInfo(shardId, null, null); processTask = new ProcessTask( shardInfo, config, mockRecordProcessor, mockCheckpointer, mockDataFetcher, taskBackoffTimeMillis); diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/SequenceNumberValidatorTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/SequenceNumberValidatorTest.java index ce222f9e..aae93f29 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/SequenceNumberValidatorTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/SequenceNumberValidatorTest.java @@ -86,9 +86,9 @@ public class SequenceNumberValidatorTest { IKinesisProxy proxy, boolean validateWithGetIterator) { - String[] nonNumericStrings = - { null, "bogus-sequence-number", SentinelCheckpoint.LATEST.toString(), - SentinelCheckpoint.SHARD_END.toString(), SentinelCheckpoint.TRIM_HORIZON.toString() }; + String[] nonNumericStrings = { null, "bogus-sequence-number", SentinelCheckpoint.LATEST.toString(), + SentinelCheckpoint.SHARD_END.toString(), SentinelCheckpoint.TRIM_HORIZON.toString(), + SentinelCheckpoint.AT_TIMESTAMP.toString() }; for (String nonNumericString : nonNumericStrings) { try { diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumerTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumerTest.java index 27f8f13c..d8d39377 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumerTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumerTest.java @@ -18,6 +18,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; import static org.mockito.Matchers.any; @@ -33,6 +34,7 @@ import static org.mockito.Mockito.when; import java.io.File; import java.math.BigInteger; import java.util.ArrayList; +import java.util.Date; import java.util.List; import java.util.ListIterator; import java.util.concurrent.ExecutionException; @@ -77,7 +79,8 @@ public class ShardConsumerTest { private final boolean cleanupLeasesOfCompletedShards = true; // We don't want any of these tests to run checkpoint validation private final boolean skipCheckpointValidationValue = false; - private final InitialPositionInStream initialPositionInStream = InitialPositionInStream.LATEST; + private static final InitialPositionInStreamExtended INITIAL_POSITION_LATEST = + InitialPositionInStreamExtended.newInitialPosition(InitialPositionInStream.LATEST); // Use Executors.newFixedThreadPool since it returns ThreadPoolExecutor, which is // ... a non-final public class, and so can be mocked and spied. @@ -102,8 +105,7 @@ public class ShardConsumerTest { 1, 10, callProcessRecordsForEmptyRecordList, - skipCheckpointValidationValue, - initialPositionInStream); + skipCheckpointValidationValue, INITIAL_POSITION_LATEST); ShardConsumer consumer = new ShardConsumer(shardInfo, @@ -153,8 +155,7 @@ public class ShardConsumerTest { 1, 10, callProcessRecordsForEmptyRecordList, - skipCheckpointValidationValue, - initialPositionInStream); + skipCheckpointValidationValue, INITIAL_POSITION_LATEST); ShardConsumer consumer = new ShardConsumer(shardInfo, @@ -198,8 +199,7 @@ public class ShardConsumerTest { 1, 10, callProcessRecordsForEmptyRecordList, - skipCheckpointValidationValue, - initialPositionInStream); + skipCheckpointValidationValue, INITIAL_POSITION_LATEST); ShardConsumer consumer = new ShardConsumer(shardInfo, @@ -287,8 +287,7 @@ public class ShardConsumerTest { maxRecords, idleTimeMS, callProcessRecordsForEmptyRecordList, - skipCheckpointValidationValue, - initialPositionInStream); + skipCheckpointValidationValue, INITIAL_POSITION_LATEST); ShardInfo shardInfo = new ShardInfo(streamShardId, testConcurrencyToken, null); ShardConsumer consumer = @@ -334,12 +333,103 @@ public class ShardConsumerTest { executorService.shutdown(); executorService.awaitTermination(60, TimeUnit.SECONDS); - String iterator = fileBasedProxy.getIterator(streamShardId, ShardIteratorType.TRIM_HORIZON.toString(), null); + String iterator = fileBasedProxy.getIterator(streamShardId, ShardIteratorType.TRIM_HORIZON.toString()); List expectedRecords = toUserRecords(fileBasedProxy.get(iterator, numRecs).getRecords()); verifyConsumedRecords(expectedRecords, processor.getProcessedRecords()); file.delete(); } + /** + * Test method for {@link com.amazonaws.services.kinesis.clientlibrary.lib.worker.ShardConsumer#consumeShard()} + * that starts from initial position of type AT_TIMESTAMP. + */ + @Test + public final void testConsumeShardWithInitialPositionAtTimestamp() throws Exception { + int numRecs = 7; + BigInteger startSeqNum = BigInteger.ONE; + Date timestamp = new Date(KinesisLocalFileDataCreator.STARTING_TIMESTAMP + 3); + InitialPositionInStreamExtended atTimestamp = + InitialPositionInStreamExtended.newInitialPositionAtTimestamp(timestamp); + String streamShardId = "kinesis-0-0"; + String testConcurrencyToken = "testToken"; + File file = + KinesisLocalFileDataCreator.generateTempDataFile(1, + "kinesis-0-", + numRecs, + startSeqNum, + "unitTestSCT002"); + + IKinesisProxy fileBasedProxy = new KinesisLocalFileProxy(file.getAbsolutePath()); + + final int maxRecords = 2; + final int idleTimeMS = 0; // keep unit tests fast + ICheckpoint checkpoint = new InMemoryCheckpointImpl(startSeqNum.toString()); + checkpoint.setCheckpoint(streamShardId, ExtendedSequenceNumber.AT_TIMESTAMP, testConcurrencyToken); + @SuppressWarnings("unchecked") + ILeaseManager leaseManager = mock(ILeaseManager.class); + when(leaseManager.getLease(anyString())).thenReturn(null); + + TestStreamlet processor = new TestStreamlet(); + + StreamConfig streamConfig = + new StreamConfig(fileBasedProxy, + maxRecords, + idleTimeMS, + callProcessRecordsForEmptyRecordList, + skipCheckpointValidationValue, + atTimestamp); + + ShardInfo shardInfo = new ShardInfo(streamShardId, testConcurrencyToken, null); + ShardConsumer consumer = + new ShardConsumer(shardInfo, + streamConfig, + checkpoint, + processor, + leaseManager, + parentShardPollIntervalMillis, + cleanupLeasesOfCompletedShards, + executorService, + metricsFactory, + taskBackoffTimeMillis); + + assertThat(consumer.getCurrentState(), is(equalTo(ShardConsumerState.WAITING_ON_PARENT_SHARDS))); + consumer.consumeShard(); // check on parent shards + Thread.sleep(50L); + consumer.consumeShard(); // start initialization + assertThat(consumer.getCurrentState(), is(equalTo(ShardConsumerState.INITIALIZING))); + consumer.consumeShard(); // initialize + Thread.sleep(50L); + + // We expect to process all records in numRecs calls + for (int i = 0; i < numRecs;) { + boolean newTaskSubmitted = consumer.consumeShard(); + if (newTaskSubmitted) { + LOG.debug("New processing task was submitted, call # " + i); + assertThat(consumer.getCurrentState(), is(equalTo(ShardConsumerState.PROCESSING))); + // CHECKSTYLE:IGNORE ModifiedControlVariable FOR NEXT 1 LINES + i += maxRecords; + } + Thread.sleep(50L); + } + + assertThat(processor.getShutdownReason(), nullValue()); + consumer.beginShutdown(); + Thread.sleep(50L); + assertThat(consumer.getCurrentState(), is(equalTo(ShardConsumerState.SHUTTING_DOWN))); + consumer.beginShutdown(); + assertThat(consumer.getCurrentState(), is(equalTo(ShardConsumerState.SHUTDOWN_COMPLETE))); + assertThat(processor.getShutdownReason(), is(equalTo(ShutdownReason.ZOMBIE))); + + executorService.shutdown(); + executorService.awaitTermination(60, TimeUnit.SECONDS); + + String iterator = fileBasedProxy.getIterator(streamShardId, timestamp); + List expectedRecords = toUserRecords(fileBasedProxy.get(iterator, numRecs).getRecords()); + verifyConsumedRecords(expectedRecords, processor.getProcessedRecords()); + assertEquals(4, processor.getProcessedRecords().size()); + file.delete(); + } + //@formatter:off (gets the formatting wrong) private void verifyConsumedRecords(List expectedRecords, List actualRecords) { diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTaskIntegrationTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTaskIntegrationTest.java index 6843efbd..307596e3 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTaskIntegrationTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTaskIntegrationTest.java @@ -120,8 +120,11 @@ public class ShardSyncTaskIntegrationTest { } leaseManager.deleteAll(); Set shardIds = kinesisProxy.getAllShardIds(); - ShardSyncTask syncTask = - new ShardSyncTask(kinesisProxy, leaseManager, InitialPositionInStream.LATEST, false, 0L); + ShardSyncTask syncTask = new ShardSyncTask(kinesisProxy, + leaseManager, + InitialPositionInStreamExtended.newInitialPosition(InitialPositionInStream.LATEST), + false, + 0L); syncTask.call(); List leases = leaseManager.listLeases(); Set leaseKeys = new HashSet(); diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncerTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncerTest.java index f02943b4..b8f6ae56 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncerTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncerTest.java @@ -18,6 +18,7 @@ import java.io.File; import java.io.IOException; import java.math.BigInteger; import java.util.ArrayList; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -59,9 +60,14 @@ import junit.framework.Assert; // CHECKSTYLE:IGNORE JavaNCSS FOR NEXT 800 LINES public class ShardSyncerTest { private static final Log LOG = LogFactory.getLog(ShardSyncer.class); - private final InitialPositionInStream latestPosition = InitialPositionInStream.LATEST; + private static final InitialPositionInStreamExtended INITIAL_POSITION_LATEST = + InitialPositionInStreamExtended.newInitialPosition(InitialPositionInStream.LATEST); + private static final InitialPositionInStreamExtended INITIAL_POSITION_TRIM_HORIZON = + InitialPositionInStreamExtended.newInitialPosition(InitialPositionInStream.TRIM_HORIZON); + private static final InitialPositionInStreamExtended INITIAL_POSITION_AT_TIMESTAMP = + InitialPositionInStreamExtended.newInitialPositionAtTimestamp(new Date(1000L)); private final boolean cleanupLeasesOfCompletedShards = true; - AmazonDynamoDB ddbClient = DynamoDBEmbedded.create(); + AmazonDynamoDB ddbClient = DynamoDBEmbedded.create().amazonDynamoDB(); LeaseManager leaseManager = new KinesisClientLeaseManager("tempTestTable", ddbClient); private static final int EXPONENT = 128; /** @@ -111,8 +117,7 @@ public class ShardSyncerTest { List shards = new ArrayList(); List leases = new ArrayList(); - Assert.assertTrue( - ShardSyncer.determineNewLeasesToCreate(shards, leases, InitialPositionInStream.LATEST).isEmpty()); + Assert.assertTrue(ShardSyncer.determineNewLeasesToCreate(shards, leases, INITIAL_POSITION_LATEST).isEmpty()); } /** @@ -131,7 +136,7 @@ public class ShardSyncerTest { shards.add(ShardObjectHelper.newShard(shardId1, null, null, sequenceRange)); List newLeases = - ShardSyncer.determineNewLeasesToCreate(shards, currentLeases, InitialPositionInStream.LATEST); + ShardSyncer.determineNewLeasesToCreate(shards, currentLeases, INITIAL_POSITION_LATEST); Assert.assertEquals(2, newLeases.size()); Set expectedLeaseShardIds = new HashSet(); expectedLeaseShardIds.add(shardId0); @@ -154,7 +159,7 @@ public class ShardSyncerTest { public final void testBootstrapShardLeasesAtTrimHorizon() throws DependencyException, InvalidStateException, ProvisionedThroughputException, IOException, KinesisClientLibIOException { - testBootstrapShardLeasesAtStartingPosition(InitialPositionInStream.TRIM_HORIZON); + testBootstrapShardLeasesAtStartingPosition(INITIAL_POSITION_TRIM_HORIZON); } /** @@ -170,7 +175,7 @@ public class ShardSyncerTest { public final void testBootstrapShardLeasesAtLatest() throws DependencyException, InvalidStateException, ProvisionedThroughputException, IOException, KinesisClientLibIOException { - testBootstrapShardLeasesAtStartingPosition(InitialPositionInStream.LATEST); + testBootstrapShardLeasesAtStartingPosition(INITIAL_POSITION_LATEST); } /** @@ -189,9 +194,7 @@ public class ShardSyncerTest { dataFile.deleteOnExit(); IKinesisProxy kinesisProxy = new KinesisLocalFileProxy(dataFile.getAbsolutePath()); - ShardSyncer.checkAndCreateLeasesForNewShards(kinesisProxy, - leaseManager, - InitialPositionInStream.LATEST, + ShardSyncer.checkAndCreateLeasesForNewShards(kinesisProxy, leaseManager, INITIAL_POSITION_LATEST, cleanupLeasesOfCompletedShards); List newLeases = leaseManager.listLeases(); Set expectedLeaseShardIds = new HashSet(); @@ -223,9 +226,7 @@ public class ShardSyncerTest { dataFile.deleteOnExit(); IKinesisProxy kinesisProxy = new KinesisLocalFileProxy(dataFile.getAbsolutePath()); - ShardSyncer.checkAndCreateLeasesForNewShards(kinesisProxy, - leaseManager, - InitialPositionInStream.TRIM_HORIZON, + ShardSyncer.checkAndCreateLeasesForNewShards(kinesisProxy, leaseManager, INITIAL_POSITION_TRIM_HORIZON, cleanupLeasesOfCompletedShards); List newLeases = leaseManager.listLeases(); Set expectedLeaseShardIds = new HashSet(); @@ -240,6 +241,37 @@ public class ShardSyncerTest { dataFile.delete(); } + /** + * @throws KinesisClientLibIOException + * @throws DependencyException + * @throws InvalidStateException + * @throws ProvisionedThroughputException + * @throws IOException + */ + @Test + public final void testCheckAndCreateLeasesForNewShardsAtTimestamp() + throws KinesisClientLibIOException, DependencyException, InvalidStateException, + ProvisionedThroughputException, IOException { + List shards = constructShardListForGraphA(); + File dataFile = KinesisLocalFileDataCreator.generateTempDataFile(shards, 1, "testBootstrap1"); + dataFile.deleteOnExit(); + IKinesisProxy kinesisProxy = new KinesisLocalFileProxy(dataFile.getAbsolutePath()); + + ShardSyncer.checkAndCreateLeasesForNewShards(kinesisProxy, leaseManager, INITIAL_POSITION_AT_TIMESTAMP, + cleanupLeasesOfCompletedShards); + List newLeases = leaseManager.listLeases(); + Set expectedLeaseShardIds = new HashSet(); + for (int i = 0; i < 11; i++) { + expectedLeaseShardIds.add("shardId-" + i); + } + Assert.assertEquals(expectedLeaseShardIds.size(), newLeases.size()); + for (KinesisClientLease lease1 : newLeases) { + Assert.assertTrue(expectedLeaseShardIds.contains(lease1.getLeaseKey())); + Assert.assertEquals(ExtendedSequenceNumber.AT_TIMESTAMP, lease1.getCheckpoint()); + } + dataFile.delete(); + } + /** * @throws KinesisClientLibIOException * @throws DependencyException @@ -259,9 +291,7 @@ public class ShardSyncerTest { dataFile.deleteOnExit(); IKinesisProxy kinesisProxy = new KinesisLocalFileProxy(dataFile.getAbsolutePath()); - ShardSyncer.checkAndCreateLeasesForNewShards(kinesisProxy, - leaseManager, - InitialPositionInStream.TRIM_HORIZON, + ShardSyncer.checkAndCreateLeasesForNewShards(kinesisProxy, leaseManager, INITIAL_POSITION_TRIM_HORIZON, cleanupLeasesOfCompletedShards); dataFile.delete(); } @@ -275,9 +305,10 @@ public class ShardSyncerTest { */ @Test public final void testCheckAndCreateLeasesForNewShardsAtTrimHorizonAndClosedShard() - throws KinesisClientLibIOException, DependencyException, InvalidStateException, ProvisionedThroughputException, - IOException { - testCheckAndCreateLeasesForNewShardsAtTrimHorizonAndClosedShardImpl(null, Integer.MAX_VALUE); + throws KinesisClientLibIOException, DependencyException, InvalidStateException, + ProvisionedThroughputException, IOException { + testCheckAndCreateLeasesForNewShardsAtSpecifiedPositionAndClosedShardImpl(null, + Integer.MAX_VALUE, INITIAL_POSITION_TRIM_HORIZON); } /** @@ -295,8 +326,8 @@ public class ShardSyncerTest { // From the Shard Graph, the max count of calling could be 10 int maxCallingCount = 10; for (int c = 1; c <= maxCallingCount; c = c + 2) { - testCheckAndCreateLeasesForNewShardsAtTrimHorizonAndClosedShardImpl( - ExceptionThrowingLeaseManagerMethods.DELETELEASE, c); + testCheckAndCreateLeasesForNewShardsAtSpecifiedPositionAndClosedShardImpl( + ExceptionThrowingLeaseManagerMethods.DELETELEASE, c, INITIAL_POSITION_TRIM_HORIZON); // Need to clean up lease manager every time after calling ShardSyncer leaseManager.deleteAll(); } @@ -317,8 +348,8 @@ public class ShardSyncerTest { // From the Shard Graph, the max count of calling could be 10 int maxCallingCount = 10; for (int c = 1; c <= maxCallingCount; c = c + 2) { - testCheckAndCreateLeasesForNewShardsAtTrimHorizonAndClosedShardImpl( - ExceptionThrowingLeaseManagerMethods.LISTLEASES, c); + testCheckAndCreateLeasesForNewShardsAtSpecifiedPositionAndClosedShardImpl( + ExceptionThrowingLeaseManagerMethods.LISTLEASES, c, INITIAL_POSITION_TRIM_HORIZON); // Need to clean up lease manager every time after calling ShardSyncer leaseManager.deleteAll(); } @@ -339,8 +370,8 @@ public class ShardSyncerTest { // From the Shard Graph, the max count of calling could be 10 int maxCallingCount = 5; for (int c = 1; c <= maxCallingCount; c = c + 2) { - testCheckAndCreateLeasesForNewShardsAtTrimHorizonAndClosedShardImpl( - ExceptionThrowingLeaseManagerMethods.CREATELEASEIFNOTEXISTS, c); + testCheckAndCreateLeasesForNewShardsAtSpecifiedPositionAndClosedShardImpl( + ExceptionThrowingLeaseManagerMethods.CREATELEASEIFNOTEXISTS, c,INITIAL_POSITION_TRIM_HORIZON); // Need to clean up lease manager every time after calling ShardSyncer leaseManager.deleteAll(); } @@ -352,7 +383,7 @@ public class ShardSyncerTest { // 2). exceptionTime is a very big or negative value. private void retryCheckAndCreateLeaseForNewShards(IKinesisProxy kinesisProxy, ExceptionThrowingLeaseManagerMethods exceptionMethod, - int exceptionTime) + int exceptionTime, InitialPositionInStreamExtended position) throws KinesisClientLibIOException, DependencyException, InvalidStateException, ProvisionedThroughputException { if (exceptionMethod != null) { ExceptionThrowingLeaseManager exceptionThrowingLeaseManager = @@ -364,7 +395,7 @@ public class ShardSyncerTest { try { ShardSyncer.checkAndCreateLeasesForNewShards(kinesisProxy, exceptionThrowingLeaseManager, - InitialPositionInStream.TRIM_HORIZON, + position, cleanupLeasesOfCompletedShards); return; } catch (LeasingException e) { @@ -376,28 +407,116 @@ public class ShardSyncerTest { } else { ShardSyncer.checkAndCreateLeasesForNewShards(kinesisProxy, leaseManager, - InitialPositionInStream.TRIM_HORIZON, + position, cleanupLeasesOfCompletedShards); } } + /** + * @throws KinesisClientLibIOException + * @throws DependencyException + * @throws InvalidStateException + * @throws ProvisionedThroughputException + * @throws IOException + */ + @Test + public final void testCheckAndCreateLeasesForNewShardsAtTimestampAndClosedShard() + throws KinesisClientLibIOException, DependencyException, InvalidStateException, + ProvisionedThroughputException, IOException { + testCheckAndCreateLeasesForNewShardsAtSpecifiedPositionAndClosedShardImpl(null, + Integer.MAX_VALUE, INITIAL_POSITION_AT_TIMESTAMP); + } + + /** + * @throws KinesisClientLibIOException + * @throws DependencyException + * @throws InvalidStateException + * @throws ProvisionedThroughputException + * @throws IOException + */ + @Test + public final void testCheckAndCreateLeasesForNewShardsAtTimestampAndClosedShardWithDeleteLeaseExceptions() + throws KinesisClientLibIOException, DependencyException, InvalidStateException, ProvisionedThroughputException, + IOException { + // Define the max calling count for lease manager methods. + // From the Shard Graph, the max count of calling could be 10 + int maxCallingCount = 10; + for (int c = 1; c <= maxCallingCount; c = c + 2) { + testCheckAndCreateLeasesForNewShardsAtSpecifiedPositionAndClosedShardImpl( + ExceptionThrowingLeaseManagerMethods.DELETELEASE, + c, INITIAL_POSITION_AT_TIMESTAMP); + // Need to clean up lease manager every time after calling ShardSyncer + leaseManager.deleteAll(); + } + } + + /** + * @throws KinesisClientLibIOException + * @throws DependencyException + * @throws InvalidStateException + * @throws ProvisionedThroughputException + * @throws IOException + */ + @Test + public final void testCheckAndCreateLeasesForNewShardsAtTimestampAndClosedShardWithListLeasesExceptions() + throws KinesisClientLibIOException, DependencyException, InvalidStateException, ProvisionedThroughputException, + IOException { + // Define the max calling count for lease manager methods. + // From the Shard Graph, the max count of calling could be 10 + int maxCallingCount = 10; + for (int c = 1; c <= maxCallingCount; c = c + 2) { + testCheckAndCreateLeasesForNewShardsAtSpecifiedPositionAndClosedShardImpl( + ExceptionThrowingLeaseManagerMethods.LISTLEASES, + c, INITIAL_POSITION_AT_TIMESTAMP); + // Need to clean up lease manager every time after calling ShardSyncer + leaseManager.deleteAll(); + } + } + + /** + * @throws KinesisClientLibIOException + * @throws DependencyException + * @throws InvalidStateException + * @throws ProvisionedThroughputException + * @throws IOException + */ + @Test + public final void testCheckAndCreateLeasesForNewShardsAtTimestampAndClosedShardWithCreateLeaseExceptions() + throws KinesisClientLibIOException, DependencyException, InvalidStateException, ProvisionedThroughputException, + IOException { + // Define the max calling count for lease manager methods. + // From the Shard Graph, the max count of calling could be 10 + int maxCallingCount = 5; + for (int c = 1; c <= maxCallingCount; c = c + 2) { + testCheckAndCreateLeasesForNewShardsAtSpecifiedPositionAndClosedShardImpl( + ExceptionThrowingLeaseManagerMethods.CREATELEASEIFNOTEXISTS, + c, INITIAL_POSITION_AT_TIMESTAMP); + // Need to clean up lease manager every time after calling ShardSyncer + leaseManager.deleteAll(); + } + } + // Real implementation of testing CheckAndCreateLeasesForNewShards with different leaseManager types. - private void testCheckAndCreateLeasesForNewShardsAtTrimHorizonAndClosedShardImpl( - ExceptionThrowingLeaseManagerMethods exceptionMethod, int exceptionTime) + private void testCheckAndCreateLeasesForNewShardsAtSpecifiedPositionAndClosedShardImpl( + ExceptionThrowingLeaseManagerMethods exceptionMethod, + int exceptionTime, + InitialPositionInStreamExtended position) throws KinesisClientLibIOException, DependencyException, InvalidStateException, ProvisionedThroughputException, IOException { + ExtendedSequenceNumber extendedSequenceNumber = + new ExtendedSequenceNumber(position.getInitialPositionInStream().toString()); List shards = constructShardListForGraphA(); File dataFile = KinesisLocalFileDataCreator.generateTempDataFile(shards, 2, "testBootstrap1"); dataFile.deleteOnExit(); IKinesisProxy kinesisProxy = new KinesisLocalFileProxy(dataFile.getAbsolutePath()); - retryCheckAndCreateLeaseForNewShards(kinesisProxy, exceptionMethod, exceptionTime); + retryCheckAndCreateLeaseForNewShards(kinesisProxy, exceptionMethod, exceptionTime, position); List newLeases = leaseManager.listLeases(); Map expectedShardIdToCheckpointMap = new HashMap(); for (int i = 0; i < 11; i++) { - expectedShardIdToCheckpointMap.put("shardId-" + i, ExtendedSequenceNumber.TRIM_HORIZON); + expectedShardIdToCheckpointMap.put("shardId-" + i, extendedSequenceNumber); } Assert.assertEquals(expectedShardIdToCheckpointMap.size(), newLeases.size()); for (KinesisClientLease lease1 : newLeases) { @@ -415,7 +534,7 @@ public class ShardSyncerTest { leaseManager.updateLease(childShardLease); expectedShardIdToCheckpointMap.put(childShardLease.getLeaseKey(), new ExtendedSequenceNumber("34290")); - retryCheckAndCreateLeaseForNewShards(kinesisProxy, exceptionMethod, exceptionTime); + retryCheckAndCreateLeaseForNewShards(kinesisProxy, exceptionMethod, exceptionTime, position); newLeases = leaseManager.listLeases(); Assert.assertEquals(expectedShardIdToCheckpointMap.size(), newLeases.size()); @@ -449,11 +568,11 @@ public class ShardSyncerTest { garbageLease.setCheckpoint(new ExtendedSequenceNumber("999")); leaseManager.createLeaseIfNotExists(garbageLease); Assert.assertEquals(garbageShardId, leaseManager.getLease(garbageShardId).getLeaseKey()); - testBootstrapShardLeasesAtStartingPosition(InitialPositionInStream.LATEST); + testBootstrapShardLeasesAtStartingPosition(INITIAL_POSITION_LATEST); Assert.assertNull(leaseManager.getLease(garbageShardId)); } - private void testBootstrapShardLeasesAtStartingPosition(InitialPositionInStream initialPosition) + private void testBootstrapShardLeasesAtStartingPosition(InitialPositionInStreamExtended initialPosition) throws DependencyException, InvalidStateException, ProvisionedThroughputException, IOException, KinesisClientLibIOException { List shards = new ArrayList(); @@ -463,7 +582,7 @@ public class ShardSyncerTest { shards.add(ShardObjectHelper.newShard(shardId0, null, null, sequenceRange)); String shardId1 = "shardId-1"; shards.add(ShardObjectHelper.newShard(shardId1, null, null, sequenceRange)); - File dataFile = KinesisLocalFileDataCreator.generateTempDataFile(shards, 10, "testBootstrap1"); + File dataFile = KinesisLocalFileDataCreator.generateTempDataFile(shards, 2, "testBootstrap1"); dataFile.deleteOnExit(); IKinesisProxy kinesisProxy = new KinesisLocalFileProxy(dataFile.getAbsolutePath()); @@ -475,7 +594,8 @@ public class ShardSyncerTest { expectedLeaseShardIds.add(shardId1); for (KinesisClientLease lease1 : newLeases) { Assert.assertTrue(expectedLeaseShardIds.contains(lease1.getLeaseKey())); - Assert.assertEquals(new ExtendedSequenceNumber(initialPosition.toString()), lease1.getCheckpoint()); + Assert.assertEquals(new ExtendedSequenceNumber(initialPosition.getInitialPositionInStream().toString()), + lease1.getCheckpoint()); } dataFile.delete(); } @@ -495,11 +615,11 @@ public class ShardSyncerTest { String shardId1 = "shardId-1"; shards.add(ShardObjectHelper.newShard(shardId1, null, null, sequenceRange)); - Set initialPositions = new HashSet(); - initialPositions.add(InitialPositionInStream.LATEST); - initialPositions.add(InitialPositionInStream.TRIM_HORIZON); + Set initialPositions = new HashSet(); + initialPositions.add(INITIAL_POSITION_LATEST); + initialPositions.add(INITIAL_POSITION_TRIM_HORIZON); - for (InitialPositionInStream initialPosition : initialPositions) { + for (InitialPositionInStreamExtended initialPosition : initialPositions) { List newLeases = ShardSyncer.determineNewLeasesToCreate(shards, currentLeases, initialPosition); Assert.assertEquals(2, newLeases.size()); @@ -508,7 +628,8 @@ public class ShardSyncerTest { expectedLeaseShardIds.add(shardId1); for (KinesisClientLease lease : newLeases) { Assert.assertTrue(expectedLeaseShardIds.contains(lease.getLeaseKey())); - Assert.assertEquals(new ExtendedSequenceNumber(initialPosition.toString()), lease.getCheckpoint()); + Assert.assertEquals(new ExtendedSequenceNumber(initialPosition.getInitialPositionInStream().toString()), + lease.getCheckpoint()); } } } @@ -532,7 +653,7 @@ public class ShardSyncerTest { ShardObjectHelper.newSequenceNumberRange("405", null))); List newLeases = - ShardSyncer.determineNewLeasesToCreate(shards, currentLeases, InitialPositionInStream.LATEST); + ShardSyncer.determineNewLeasesToCreate(shards, currentLeases, INITIAL_POSITION_LATEST); Assert.assertEquals(1, newLeases.size()); Assert.assertEquals(lastShardId, newLeases.get(0).getLeaseKey()); } @@ -557,7 +678,7 @@ public class ShardSyncerTest { currentLeases.add(newLease("shardId-5")); List newLeases = - ShardSyncer.determineNewLeasesToCreate(shards, currentLeases, InitialPositionInStream.LATEST); + ShardSyncer.determineNewLeasesToCreate(shards, currentLeases, INITIAL_POSITION_LATEST); Map expectedShardIdCheckpointMap = new HashMap(); expectedShardIdCheckpointMap.put("shardId-8", ExtendedSequenceNumber.TRIM_HORIZON); @@ -595,7 +716,7 @@ public class ShardSyncerTest { currentLeases.add(newLease("shardId-7")); List newLeases = - ShardSyncer.determineNewLeasesToCreate(shards, currentLeases, InitialPositionInStream.LATEST); + ShardSyncer.determineNewLeasesToCreate(shards, currentLeases, INITIAL_POSITION_LATEST); Map expectedShardIdCheckpointMap = new HashMap(); expectedShardIdCheckpointMap.put("shardId-8", ExtendedSequenceNumber.TRIM_HORIZON); @@ -631,7 +752,7 @@ public class ShardSyncerTest { currentLeases.add(newLease("shardId-5")); List newLeases = - ShardSyncer.determineNewLeasesToCreate(shards, currentLeases, InitialPositionInStream.TRIM_HORIZON); + ShardSyncer.determineNewLeasesToCreate(shards, currentLeases, INITIAL_POSITION_TRIM_HORIZON); Map expectedShardIdCheckpointMap = new HashMap(); expectedShardIdCheckpointMap.put("shardId-8", ExtendedSequenceNumber.TRIM_HORIZON); @@ -671,7 +792,7 @@ public class ShardSyncerTest { currentLeases.add(newLease("shardId-7")); List newLeases = - ShardSyncer.determineNewLeasesToCreate(shards, currentLeases, InitialPositionInStream.TRIM_HORIZON); + ShardSyncer.determineNewLeasesToCreate(shards, currentLeases, INITIAL_POSITION_TRIM_HORIZON); Map expectedShardIdCheckpointMap = new HashMap(); expectedShardIdCheckpointMap.put("shardId-8", ExtendedSequenceNumber.TRIM_HORIZON); @@ -700,7 +821,7 @@ public class ShardSyncerTest { List shards = constructShardListForGraphB(); List currentLeases = new ArrayList(); List newLeases = - ShardSyncer.determineNewLeasesToCreate(shards, currentLeases, InitialPositionInStream.TRIM_HORIZON); + ShardSyncer.determineNewLeasesToCreate(shards, currentLeases, INITIAL_POSITION_TRIM_HORIZON); Map expectedShardIdCheckpointMap = new HashMap(); for (int i = 0; i < 11; i++) { @@ -716,6 +837,110 @@ public class ShardSyncerTest { } } + /** + * Test CheckIfDescendantAndAddNewLeasesForAncestors (initial position AT_TIMESTAMP) + * Shard structure (each level depicts a stream segment): + * 0 1 2 3 4 5- shards till epoch 102 + * \ / \ / | | + * 6 7 4 5- shards from epoch 103 - 205 + * \ / | /\ + * 8 4 9 10 - shards from epoch 206 (open - no ending sequenceNumber) + * Current leases: (3, 4, 5) + */ + @Test + public final void testDetermineNewLeasesToCreateSplitMergeAtTimestamp1() { + List shards = constructShardListForGraphA(); + List currentLeases = new ArrayList(); + + + currentLeases.add(newLease("shardId-3")); + currentLeases.add(newLease("shardId-4")); + currentLeases.add(newLease("shardId-5")); + + List newLeases = + ShardSyncer.determineNewLeasesToCreate(shards, currentLeases, INITIAL_POSITION_AT_TIMESTAMP); + Map expectedShardIdCheckpointMap = new HashMap(); + expectedShardIdCheckpointMap.put("shardId-8", ExtendedSequenceNumber.AT_TIMESTAMP); + expectedShardIdCheckpointMap.put("shardId-9", ExtendedSequenceNumber.AT_TIMESTAMP); + expectedShardIdCheckpointMap.put("shardId-10", ExtendedSequenceNumber.AT_TIMESTAMP); + expectedShardIdCheckpointMap.put("shardId-6", ExtendedSequenceNumber.AT_TIMESTAMP); + expectedShardIdCheckpointMap.put("shardId-2", ExtendedSequenceNumber.AT_TIMESTAMP); + expectedShardIdCheckpointMap.put("shardId-7", ExtendedSequenceNumber.AT_TIMESTAMP); + expectedShardIdCheckpointMap.put("shardId-0", ExtendedSequenceNumber.AT_TIMESTAMP); + expectedShardIdCheckpointMap.put("shardId-1", ExtendedSequenceNumber.AT_TIMESTAMP); + + Assert.assertEquals(expectedShardIdCheckpointMap.size(), newLeases.size()); + for (KinesisClientLease lease : newLeases) { + Assert.assertTrue("Unexpected lease: " + lease, + expectedShardIdCheckpointMap.containsKey(lease.getLeaseKey())); + Assert.assertEquals(expectedShardIdCheckpointMap.get(lease.getLeaseKey()), lease.getCheckpoint()); + } + } + + /** + * Test CheckIfDescendantAndAddNewLeasesForAncestors (initial position AT_TIMESTAMP) + * Shard structure (each level depicts a stream segment): + * 0 1 2 3 4 5- shards till epoch 102 + * \ / \ / | | + * 6 7 4 5- shards from epoch 103 - 205 + * \ / | /\ + * 8 4 9 10 - shards from epoch 206 (open - no ending sequenceNumber) + * Current leases: (4, 5, 7) + */ + @Test + public final void testDetermineNewLeasesToCreateSplitMergeAtTimestamp2() { + List shards = constructShardListForGraphA(); + List currentLeases = new ArrayList(); + + currentLeases.add(newLease("shardId-4")); + currentLeases.add(newLease("shardId-5")); + currentLeases.add(newLease("shardId-7")); + + List newLeases = + ShardSyncer.determineNewLeasesToCreate(shards, currentLeases, INITIAL_POSITION_AT_TIMESTAMP); + Map expectedShardIdCheckpointMap = new HashMap(); + expectedShardIdCheckpointMap.put("shardId-8", ExtendedSequenceNumber.AT_TIMESTAMP); + expectedShardIdCheckpointMap.put("shardId-9", ExtendedSequenceNumber.AT_TIMESTAMP); + expectedShardIdCheckpointMap.put("shardId-10", ExtendedSequenceNumber.AT_TIMESTAMP); + expectedShardIdCheckpointMap.put("shardId-6", ExtendedSequenceNumber.AT_TIMESTAMP); + expectedShardIdCheckpointMap.put("shardId-0", ExtendedSequenceNumber.AT_TIMESTAMP); + expectedShardIdCheckpointMap.put("shardId-1", ExtendedSequenceNumber.AT_TIMESTAMP); + + Assert.assertEquals(expectedShardIdCheckpointMap.size(), newLeases.size()); + for (KinesisClientLease lease : newLeases) { + Assert.assertTrue("Unexpected lease: " + lease, + expectedShardIdCheckpointMap.containsKey(lease.getLeaseKey())); + Assert.assertEquals(expectedShardIdCheckpointMap.get(lease.getLeaseKey()), lease.getCheckpoint()); + } + } + + /** + * Test CheckIfDescendantAndAddNewLeasesForAncestors (initial position AT_TIMESTAMP) + * For shard graph B (see the construct method doc for structure). + * Current leases: empty set + */ + @Test + public final void testDetermineNewLeasesToCreateGraphBNoInitialLeasesAtTimestamp() { + List shards = constructShardListForGraphB(); + List currentLeases = new ArrayList(); + List newLeases = + ShardSyncer.determineNewLeasesToCreate(shards, currentLeases, INITIAL_POSITION_AT_TIMESTAMP); + Map expectedShardIdCheckpointMap = + new HashMap(); + for (int i = 0; i < shards.size(); i++) { + String expectedShardId = "shardId-" + i; + expectedShardIdCheckpointMap.put(expectedShardId, ExtendedSequenceNumber.AT_TIMESTAMP); + } + + Assert.assertEquals(expectedShardIdCheckpointMap.size(), newLeases.size()); + for (KinesisClientLease lease : newLeases) { + Assert.assertTrue("Unexpected lease: " + lease, + expectedShardIdCheckpointMap.containsKey(lease.getLeaseKey())); + Assert.assertEquals(expectedShardIdCheckpointMap.get(lease.getLeaseKey()), lease.getCheckpoint()); + } + } + + /* * Helper method to construct a shard list for graph A. Graph A is defined below. * Shard structure (y-axis is epochs): @@ -808,8 +1033,7 @@ public class ShardSyncerTest { @Test public final void testCheckIfDescendantAndAddNewLeasesForAncestorsNullShardId() { Map memoizationContext = new HashMap<>(); - Assert.assertFalse(ShardSyncer.checkIfDescendantAndAddNewLeasesForAncestors(null, - latestPosition, + Assert.assertFalse(ShardSyncer.checkIfDescendantAndAddNewLeasesForAncestors(null, INITIAL_POSITION_LATEST, null, null, null, @@ -824,8 +1048,7 @@ public class ShardSyncerTest { String shardId = "shardId-trimmed"; Map kinesisShards = new HashMap(); Map memoizationContext = new HashMap<>(); - Assert.assertFalse(ShardSyncer.checkIfDescendantAndAddNewLeasesForAncestors(shardId, - latestPosition, + Assert.assertFalse(ShardSyncer.checkIfDescendantAndAddNewLeasesForAncestors(shardId, INITIAL_POSITION_LATEST, null, kinesisShards, null, @@ -844,8 +1067,7 @@ public class ShardSyncerTest { shardIdsOfCurrentLeases.add(shardId); Map newLeaseMap = new HashMap(); Map memoizationContext = new HashMap<>(); - Assert.assertTrue(ShardSyncer.checkIfDescendantAndAddNewLeasesForAncestors(shardId, - latestPosition, + Assert.assertTrue(ShardSyncer.checkIfDescendantAndAddNewLeasesForAncestors(shardId, INITIAL_POSITION_LATEST, shardIdsOfCurrentLeases, kinesisShards, newLeaseMap, @@ -872,8 +1094,7 @@ public class ShardSyncerTest { kinesisShards.put(shardId, ShardObjectHelper.newShard(shardId, parentShardId, adjacentParentShardId, null)); Map memoizationContext = new HashMap<>(); - Assert.assertFalse(ShardSyncer.checkIfDescendantAndAddNewLeasesForAncestors(shardId, - latestPosition, + Assert.assertFalse(ShardSyncer.checkIfDescendantAndAddNewLeasesForAncestors(shardId, INITIAL_POSITION_LATEST, shardIdsOfCurrentLeases, kinesisShards, newLeaseMap, @@ -902,8 +1123,7 @@ public class ShardSyncerTest { kinesisShards.put(shardId, shard); Map memoizationContext = new HashMap<>(); - Assert.assertTrue(ShardSyncer.checkIfDescendantAndAddNewLeasesForAncestors(shardId, - latestPosition, + Assert.assertTrue(ShardSyncer.checkIfDescendantAndAddNewLeasesForAncestors(shardId, INITIAL_POSITION_LATEST, shardIdsOfCurrentLeases, kinesisShards, newLeaseMap, diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTaskTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTaskTest.java index 6b77f818..a2302ad0 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTaskTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTaskTest.java @@ -42,6 +42,9 @@ import com.amazonaws.services.kinesis.leases.interfaces.ILeaseManager; */ public class ShutdownTaskTest { private static final long TASK_BACKOFF_TIME_MILLIS = 1L; + private static final InitialPositionInStreamExtended INITIAL_POSITION_TRIM_HORIZON = + InitialPositionInStreamExtended.newInitialPosition(InitialPositionInStream.TRIM_HORIZON); + Set defaultParentShardIds = new HashSet<>(); String defaultConcurrencyToken = "testToken4398"; String defaultShardId = "shardId-0000397840"; @@ -88,16 +91,15 @@ public class ShutdownTaskTest { IKinesisProxy kinesisProxy = mock(IKinesisProxy.class); ILeaseManager leaseManager = mock(KinesisClientLeaseManager.class); boolean cleanupLeasesOfCompletedShards = false; - ShutdownTask task = - new ShutdownTask(defaultShardInfo, - defaultRecordProcessor, - checkpointer, - ShutdownReason.TERMINATE, - kinesisProxy, - InitialPositionInStream.TRIM_HORIZON, - cleanupLeasesOfCompletedShards , - leaseManager, - TASK_BACKOFF_TIME_MILLIS); + ShutdownTask task = new ShutdownTask(defaultShardInfo, + defaultRecordProcessor, + checkpointer, + ShutdownReason.TERMINATE, + kinesisProxy, + INITIAL_POSITION_TRIM_HORIZON, + cleanupLeasesOfCompletedShards, + leaseManager, + TASK_BACKOFF_TIME_MILLIS); TaskResult result = task.call(); Assert.assertNotNull(result.getException()); Assert.assertTrue(result.getException() instanceof IllegalArgumentException); @@ -114,16 +116,15 @@ public class ShutdownTaskTest { when(kinesisProxy.getShardList()).thenReturn(null); ILeaseManager leaseManager = mock(KinesisClientLeaseManager.class); boolean cleanupLeasesOfCompletedShards = false; - ShutdownTask task = - new ShutdownTask(defaultShardInfo, - defaultRecordProcessor, - checkpointer, - ShutdownReason.TERMINATE, - kinesisProxy, - InitialPositionInStream.TRIM_HORIZON, - cleanupLeasesOfCompletedShards , - leaseManager, - TASK_BACKOFF_TIME_MILLIS); + ShutdownTask task = new ShutdownTask(defaultShardInfo, + defaultRecordProcessor, + checkpointer, + ShutdownReason.TERMINATE, + kinesisProxy, + INITIAL_POSITION_TRIM_HORIZON, + cleanupLeasesOfCompletedShards, + leaseManager, + TASK_BACKOFF_TIME_MILLIS); TaskResult result = task.call(); Assert.assertNotNull(result.getException()); Assert.assertTrue(result.getException() instanceof KinesisClientLibIOException); diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/WorkerTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/WorkerTest.java index f4cd9307..f0b42671 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/WorkerTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/WorkerTest.java @@ -29,6 +29,7 @@ import java.lang.Thread.State; import java.math.BigInteger; import java.util.ArrayList; import java.util.Collections; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -80,7 +81,6 @@ import com.amazonaws.services.kinesis.model.HashKeyRange; import com.amazonaws.services.kinesis.model.Record; import com.amazonaws.services.kinesis.model.SequenceNumberRange; import com.amazonaws.services.kinesis.model.Shard; -import com.amazonaws.services.kinesis.model.ShardIteratorType; /** * Unit tests of Worker. @@ -101,7 +101,10 @@ public class WorkerTest { private final boolean cleanupLeasesUponShardCompletion = true; // We don't want any of these tests to run checkpoint validation private final boolean skipCheckpointValidationValue = false; - private final InitialPositionInStream initialPositionInStream = InitialPositionInStream.LATEST; + private static final InitialPositionInStreamExtended INITIAL_POSITION_LATEST = + InitialPositionInStreamExtended.newInitialPosition(InitialPositionInStream.LATEST); + private static final InitialPositionInStreamExtended INITIAL_POSITION_TRIM_HORIZON = + InitialPositionInStreamExtended.newInitialPosition(InitialPositionInStream.TRIM_HORIZON); // CHECKSTYLE:IGNORE AnonInnerLengthCheck FOR NEXT 50 LINES private static final com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorFactory SAMPLE_RECORD_PROCESSOR_FACTORY = @@ -167,9 +170,7 @@ public class WorkerTest { new StreamConfig(proxy, maxRecords, idleTimeInMilliseconds, - callProcessRecordsForEmptyRecordList, - skipCheckpointValidationValue, - initialPositionInStream); + callProcessRecordsForEmptyRecordList, skipCheckpointValidationValue, INITIAL_POSITION_LATEST); final String testConcurrencyToken = "testToken"; final String anotherConcurrencyToken = "anotherTestToken"; final String dummyKinesisShardId = "kinesis-0-0"; @@ -182,9 +183,7 @@ public class WorkerTest { Worker worker = new Worker(stageName, - streamletFactory, - streamConfig, - InitialPositionInStream.LATEST, + streamletFactory, streamConfig, INITIAL_POSITION_LATEST, parentShardPollIntervalMillis, shardSyncIntervalMillis, cleanupLeasesUponShardCompletion, @@ -219,9 +218,7 @@ public class WorkerTest { new StreamConfig(proxy, maxRecords, idleTimeInMilliseconds, - callProcessRecordsForEmptyRecordList, - skipCheckpointValidationValue, - initialPositionInStream); + callProcessRecordsForEmptyRecordList, skipCheckpointValidationValue, INITIAL_POSITION_LATEST); final String concurrencyToken = "testToken"; final String anotherConcurrencyToken = "anotherTestToken"; final String dummyKinesisShardId = "kinesis-0-0"; @@ -235,9 +232,7 @@ public class WorkerTest { Worker worker = new Worker(stageName, - streamletFactory, - streamConfig, - InitialPositionInStream.LATEST, + streamletFactory, streamConfig, INITIAL_POSITION_LATEST, parentShardPollIntervalMillis, shardSyncIntervalMillis, cleanupLeasesUponShardCompletion, @@ -283,9 +278,7 @@ public class WorkerTest { new StreamConfig(proxy, maxRecords, idleTimeInMilliseconds, - callProcessRecordsForEmptyRecordList, - skipCheckpointValidationValue, - initialPositionInStream); + callProcessRecordsForEmptyRecordList, skipCheckpointValidationValue, INITIAL_POSITION_LATEST); KinesisClientLibLeaseCoordinator leaseCoordinator = mock(KinesisClientLibLeaseCoordinator.class); @SuppressWarnings("unchecked") ILeaseManager leaseManager = mock(ILeaseManager.class); @@ -295,8 +288,7 @@ public class WorkerTest { Worker worker = new Worker(stageName, recordProcessorFactory, - streamConfig, - InitialPositionInStream.TRIM_HORIZON, + streamConfig, INITIAL_POSITION_TRIM_HORIZON, shardPollInterval, shardSyncIntervalMillis, cleanupLeasesUponShardCompletion, @@ -621,6 +613,7 @@ public class WorkerTest { /** * Returns executor service that will be owned by the worker. This is useful to test the scenario * where worker shuts down the executor service also during shutdown flow. + * * @return Executor service that will be owned by the worker. */ private WorkerThreadPoolExecutor getWorkerThreadPoolExecutor() { @@ -665,7 +658,7 @@ public class WorkerTest { List initialLeases = new ArrayList(); for (Shard shard : shardList) { KinesisClientLease lease = ShardSyncer.newKCLLease(shard); - lease.setCheckpoint(ExtendedSequenceNumber.TRIM_HORIZON); + lease.setCheckpoint(ExtendedSequenceNumber.AT_TIMESTAMP); initialLeases.add(lease); } runAndTestWorker(shardList, threadPoolSize, initialLeases, callProcessRecordsForEmptyRecordList, numberOfRecordsPerShard); @@ -719,7 +712,7 @@ public class WorkerTest { final long epsilonMillis = 1000L; final long idleTimeInMilliseconds = 2L; - AmazonDynamoDB ddbClient = DynamoDBEmbedded.create(); + AmazonDynamoDB ddbClient = DynamoDBEmbedded.create().amazonDynamoDB(); LeaseManager leaseManager = new KinesisClientLeaseManager("foo", ddbClient); leaseManager.createLeaseTableIfNotExists(1L, 1L); for (KinesisClientLease initialLease : initialLeases) { @@ -733,19 +726,17 @@ public class WorkerTest { epsilonMillis, metricsFactory); - StreamConfig streamConfig = - new StreamConfig(kinesisProxy, - maxRecords, - idleTimeInMilliseconds, - callProcessRecordsForEmptyRecordList, - skipCheckpointValidationValue, - initialPositionInStream); + final Date timestamp = new Date(KinesisLocalFileDataCreator.STARTING_TIMESTAMP); + StreamConfig streamConfig = new StreamConfig(kinesisProxy, + maxRecords, + idleTimeInMilliseconds, + callProcessRecordsForEmptyRecordList, + skipCheckpointValidationValue, InitialPositionInStreamExtended.newInitialPositionAtTimestamp(timestamp)); Worker worker = new Worker(stageName, recordProcessorFactory, - streamConfig, - InitialPositionInStream.TRIM_HORIZON, + streamConfig, INITIAL_POSITION_TRIM_HORIZON, parentShardPollIntervalMillis, shardSyncIntervalMillis, cleanupLeasesUponShardCompletion, @@ -843,7 +834,8 @@ public class WorkerTest { findShardIdsAndStreamLetsOfShardsWithOnlyOneProcessor(recordProcessorFactory); for (Shard shard : shardList) { String shardId = shard.getShardId(); - String iterator = fileBasedProxy.getIterator(shardId, ShardIteratorType.TRIM_HORIZON.toString(), null); + String iterator = + fileBasedProxy.getIterator(shardId, new Date(KinesisLocalFileDataCreator.STARTING_TIMESTAMP)); List expectedRecords = fileBasedProxy.get(iterator, numRecs).getRecords(); if (shardIdsAndStreamLetsOfShardsWithOnlyOneProcessor.containsKey(shardId)) { verifyAllRecordsWereConsumedExactlyOnce(expectedRecords, @@ -859,7 +851,8 @@ public class WorkerTest { Map> shardStreamletsRecords) { for (Shard shard : shardList) { String shardId = shard.getShardId(); - String iterator = fileBasedProxy.getIterator(shardId, ShardIteratorType.TRIM_HORIZON.toString(), null); + String iterator = + fileBasedProxy.getIterator(shardId, new Date(KinesisLocalFileDataCreator.STARTING_TIMESTAMP)); List expectedRecords = fileBasedProxy.get(iterator, numRecs).getRecords(); verifyAllRecordsWereConsumedAtLeastOnce(expectedRecords, shardStreamletsRecords.get(shardId)); } diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisLocalFileProxy.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisLocalFileProxy.java index a346b5c6..db70b5de 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisLocalFileProxy.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisLocalFileProxy.java @@ -25,6 +25,7 @@ import java.nio.charset.Charset; import java.nio.charset.CharsetEncoder; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; @@ -65,7 +66,9 @@ public class KinesisLocalFileProxy implements IKinesisProxy { /** Partition key associated with data record. */ PARTITION_KEY(2), /** Data. */ - DATA(3); + DATA(3), + /** Approximate arrival timestamp. */ + APPROXIMATE_ARRIVAL_TIMESTAMP(4); private final int position; @@ -149,7 +152,7 @@ public class KinesisLocalFileProxy implements IKinesisProxy { String[] strArr = str.split(","); if (strArr.length != NUM_FIELDS_IN_FILE) { throw new InvalidArgumentException("Unexpected input in file." - + "Expected format (shardId, sequenceNumber, partitionKey, dataRecord)"); + + "Expected format (shardId, sequenceNumber, partitionKey, dataRecord, timestamp)"); } String shardId = strArr[LocalFileFields.SHARD_ID.getPosition()]; Record record = new Record(); @@ -157,6 +160,9 @@ public class KinesisLocalFileProxy implements IKinesisProxy { record.setPartitionKey(strArr[LocalFileFields.PARTITION_KEY.getPosition()]); ByteBuffer byteBuffer = encoder.encode(CharBuffer.wrap(strArr[LocalFileFields.DATA.getPosition()])); record.setData(byteBuffer); + Date timestamp = + new Date(Long.parseLong(strArr[LocalFileFields.APPROXIMATE_ARRIVAL_TIMESTAMP.getPosition()])); + record.setApproximateArrivalTimestamp(timestamp); List shardRecords = shardedDataRecords.get(shardId); if (shardRecords == null) { shardRecords = new ArrayList(); @@ -221,11 +227,8 @@ public class KinesisLocalFileProxy implements IKinesisProxy { return new IteratorInfo(splits[0], splits[1]); } - /* - * (non-Javadoc) - * - * @see com.amazonaws.services.kinesis.clientlibrary.proxies.IKinesisProxy#getIterator(java.lang.String, - * java.lang.String, java.lang.String) + /** + * {@inheritDoc} */ @Override public String getIterator(String shardId, String iteratorEnum, String sequenceNumber) @@ -262,6 +265,77 @@ public class KinesisLocalFileProxy implements IKinesisProxy { } } + /** + * {@inheritDoc} + */ + @Override + public String getIterator(String shardId, String iteratorEnum) + throws ResourceNotFoundException, InvalidArgumentException { + /* + * If we don't have records in this shard, any iterator will return the empty list. Using a + * sequence number of 1 on an empty shard will give this behavior. + */ + List shardRecords = shardedDataRecords.get(shardId); + if (shardRecords == null) { + throw new ResourceNotFoundException(shardId + " does not exist"); + } + if (shardRecords.isEmpty()) { + return serializeIterator(shardId, "1"); + } + + final String serializedIterator; + if (ShardIteratorType.LATEST.toString().equals(iteratorEnum)) { + /* + * If we do have records, LATEST should return an iterator that can be used to read the + * last record. Our iterators are inclusive for convenience. + */ + Record last = shardRecords.get(shardRecords.size() - 1); + serializedIterator = serializeIterator(shardId, last.getSequenceNumber()); + } else if (ShardIteratorType.TRIM_HORIZON.toString().equals(iteratorEnum)) { + serializedIterator = serializeIterator(shardId, shardRecords.get(0).getSequenceNumber()); + } else { + throw new IllegalArgumentException("IteratorEnum value was invalid: " + iteratorEnum); + } + return serializedIterator; + } + + /** + * {@inheritDoc} + */ + @Override + public String getIterator(String shardId, Date timestamp) + throws ResourceNotFoundException, InvalidArgumentException { + /* + * If we don't have records in this shard, any iterator will return the empty list. Using a + * sequence number of 1 on an empty shard will give this behavior. + */ + List shardRecords = shardedDataRecords.get(shardId); + if (shardRecords == null) { + throw new ResourceNotFoundException(shardId + " does not exist"); + } + if (shardRecords.isEmpty()) { + return serializeIterator(shardId, "1"); + } + + final String serializedIterator; + if (timestamp != null) { + String seqNumAtTimestamp = findSequenceNumberAtTimestamp(shardRecords, timestamp); + serializedIterator = serializeIterator(shardId, seqNumAtTimestamp); + } else { + throw new IllegalArgumentException("Timestamp must be specified for AT_TIMESTAMP iterator"); + } + return serializedIterator; + } + + private String findSequenceNumberAtTimestamp(final List shardRecords, final Date timestamp) { + for (Record rec : shardRecords) { + if (rec.getApproximateArrivalTimestamp().getTime() >= timestamp.getTime()) { + return rec.getSequenceNumber(); + } + } + return null; + } + /* * (non-Javadoc) * diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/util/KinesisLocalFileDataCreator.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/util/KinesisLocalFileDataCreator.java index 795f2db9..e5e4419a 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/util/KinesisLocalFileDataCreator.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/util/KinesisLocalFileDataCreator.java @@ -51,6 +51,17 @@ public class KinesisLocalFileDataCreator { private static final int PARTITION_KEY_LENGTH = 10; private static final int DATA_LENGTH = 40; + /** + * Starting timestamp - also referenced in KinesisLocalFileProxyTest. + */ + public static final long STARTING_TIMESTAMP = 1462345678910L; + + /** + * This is used to allow few records to have the same timestamps (to mimic real life scenarios). + * Records 5n-1 and 5n will have the same timestamp (n > 0). + */ + private static final int DIVISOR = 5; + private KinesisLocalFileDataCreator() { } @@ -96,6 +107,7 @@ public class KinesisLocalFileDataCreator { fileWriter.write(serializedShardList); fileWriter.newLine(); BigInteger sequenceNumberIncrement = new BigInteger("0"); + long timestamp = STARTING_TIMESTAMP; for (int i = 0; i < numRecordsPerShard; i++) { for (Shard shard : shardList) { BigInteger sequenceNumber = @@ -112,7 +124,12 @@ public class KinesisLocalFileDataCreator { String partitionKey = PARTITION_KEY_PREFIX + shard.getShardId() + generateRandomString(PARTITION_KEY_LENGTH); String data = generateRandomString(DATA_LENGTH); - String line = shard.getShardId() + "," + sequenceNumber + "," + partitionKey + "," + data; + + // Allow few records to have the same timestamps (to mimic real life scenarios). + timestamp = (i % DIVISOR == 0) ? timestamp : timestamp + 1; + String line = shard.getShardId() + "," + sequenceNumber + "," + partitionKey + "," + data + "," + + timestamp; + fileWriter.write(line); fileWriter.newLine(); sequenceNumberIncrement = sequenceNumberIncrement.add(BigInteger.ONE); From 85b6b9f1510875a539bfcdf4920f534635d709dc Mon Sep 17 00:00:00 2001 From: "Pfifer, Justin" Date: Wed, 17 Aug 2016 07:45:33 -0700 Subject: [PATCH 5/9] Allow Prioritization of Parent Shard Tasks Added a new interface that allows the worker to prioritize which lease assignment it will work on next. When using the ParentsFirstshardprioritization the worker will select parents for processing before selecting children. This will prevent ShardConsumers from spending time sleeping in the WAITING_ON_PARENT_SHARDS state. --- .../worker/KinesisClientLibConfiguration.java | 25 +++ .../KinesisClientLibLeaseCoordinator.java | 3 +- .../lib/worker/NoOpShardPrioritization.java | 21 +++ .../ParentsFirstShardPrioritization.java | 135 +++++++++++++++ .../lib/worker/ShardConsumer.java | 39 ++--- .../clientlibrary/lib/worker/ShardInfo.java | 64 ++++++- .../lib/worker/ShardPrioritization.java | 19 ++ .../clientlibrary/lib/worker/Worker.java | 38 +++- .../worker/BlockOnParentShardTaskTest.java | 15 +- .../lib/worker/KinesisDataFetcherTest.java | 2 +- ...rentsFirstShardPrioritizationUnitTest.java | 162 ++++++++++++++++++ .../lib/worker/ProcessTaskTest.java | 2 +- .../RecordProcessorCheckpointerTest.java | 16 +- .../lib/worker/ShardConsumerTest.java | 10 +- .../lib/worker/ShardInfoTest.java | 33 ++-- .../lib/worker/ShutdownTaskTest.java | 6 +- .../clientlibrary/lib/worker/WorkerTest.java | 23 ++- 17 files changed, 536 insertions(+), 77 deletions(-) create mode 100644 src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/NoOpShardPrioritization.java create mode 100644 src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ParentsFirstShardPrioritization.java create mode 100644 src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardPrioritization.java create mode 100644 src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ParentsFirstShardPrioritizationUnitTest.java diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java index 9a6fb4f5..590382b4 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java @@ -154,6 +154,10 @@ public class KinesisClientLibConfiguration { */ public static final int DEFAULT_INITIAL_LEASE_TABLE_WRITE_CAPACITY = 10; + /** + * Default Shard prioritization strategy. + */ + public static final ShardPrioritization DEFAULT_SHARD_PRIORITIZATION = new NoOpShardPrioritization(); private String applicationName; private String tableName; @@ -187,6 +191,7 @@ public class KinesisClientLibConfiguration { private int initialLeaseTableReadCapacity; private int initialLeaseTableWriteCapacity; private InitialPositionInStreamExtended initialPositionInStreamExtended; + private ShardPrioritization shardPrioritization; /** * Constructor. @@ -333,6 +338,7 @@ public class KinesisClientLibConfiguration { this.initialLeaseTableWriteCapacity = DEFAULT_INITIAL_LEASE_TABLE_WRITE_CAPACITY; this.initialPositionInStreamExtended = InitialPositionInStreamExtended.newInitialPosition(initialPositionInStream); + this.shardPrioritization = DEFAULT_SHARD_PRIORITIZATION; } // Check if value is positive, otherwise throw an exception @@ -599,6 +605,13 @@ public class KinesisClientLibConfiguration { return initialPositionInStreamExtended.getTimestamp(); } + /** + * @return Shard prioritization strategy. + */ + public ShardPrioritization getShardPrioritizationStrategy() { + return shardPrioritization; + } + // CHECKSTYLE:IGNORE HiddenFieldCheck FOR NEXT 190 LINES /** * @param tableName name of the lease table in DynamoDB @@ -913,4 +926,16 @@ public class KinesisClientLibConfiguration { this.initialLeaseTableWriteCapacity = initialLeaseTableWriteCapacity; return this; } + + /** + * @param shardPrioritization Implementation of ShardPrioritization interface that should be used during processing. + * @return KinesisClientLibConfiguration + */ + public KinesisClientLibConfiguration withShardPrioritizationStrategy(ShardPrioritization shardPrioritization) { + if (shardPrioritization == null) { + throw new IllegalArgumentException("shardPrioritization cannot be null"); + } + this.shardPrioritization = shardPrioritization; + return this; + } } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibLeaseCoordinator.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibLeaseCoordinator.java index 20ca4d90..0119dcc3 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibLeaseCoordinator.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibLeaseCoordinator.java @@ -209,7 +209,8 @@ class KinesisClientLibLeaseCoordinator extends LeaseCoordinator prioritize(List original) { + return original; + } +} diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ParentsFirstShardPrioritization.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ParentsFirstShardPrioritization.java new file mode 100644 index 00000000..dbacbd98 --- /dev/null +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ParentsFirstShardPrioritization.java @@ -0,0 +1,135 @@ +package com.amazonaws.services.kinesis.clientlibrary.lib.worker; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Shard Prioritization that prioritizes parent shards first. + * It also limits number of shards that will be available for initialization based on their depth. + * It doesn't make a lot of sense to work on a shard that has too many unfinished parents. + */ +public class ParentsFirstShardPrioritization implements + ShardPrioritization { + private static final SortingNode PROCESSING_NODE = new SortingNode(null, Integer.MIN_VALUE); + + private final int maxDepth; + + /** + * Creates ParentFirst prioritization with filtering based on depth of the shard. + * Shards that have depth > maxDepth will be ignored and will not be returned by this prioritization. + * + * @param maxDepth any shard that is deeper than max depth, will be excluded from processing + */ + public ParentsFirstShardPrioritization(int maxDepth) { + /* Depth 0 means that shard is completed or cannot be found, + * it is impossible to process such shards. + */ + if (maxDepth <= 0) { + throw new IllegalArgumentException("Max depth cannot be negative or zero. Provided value: " + maxDepth); + } + this.maxDepth = maxDepth; + } + + @Override + public List prioritize(List original) { + Map shards = new HashMap<>(); + for (ShardInfo shardInfo : original) { + shards.put(shardInfo.getShardId(), + shardInfo); + } + + Map processedNodes = new HashMap<>(); + + for (ShardInfo shardInfo : original) { + populateDepth(shardInfo.getShardId(), + shards, + processedNodes); + } + + List orderedInfos = new ArrayList<>(original.size()); + + List orderedNodes = new ArrayList<>(processedNodes.values()); + Collections.sort(orderedNodes); + + for (SortingNode sortingTreeNode : orderedNodes) { + // don't process shards with depth > maxDepth + if (sortingTreeNode.getDepth() <= maxDepth) { + orderedInfos.add(sortingTreeNode.shardInfo); + } + } + return orderedInfos; + } + + private int populateDepth(String shardId, + Map shards, + Map processedNodes) { + SortingNode processed = processedNodes.get(shardId); + if (processed != null) { + if (processed == PROCESSING_NODE) { + throw new IllegalArgumentException("Circular dependency detected. Shard Id " + + shardId + " is processed twice"); + } + return processed.getDepth(); + } + + ShardInfo shardInfo = shards.get(shardId); + if (shardInfo == null) { + // parent doesn't exist in our list, so this shard is root-level node + return 0; + } + + if (shardInfo.isCompleted()) { + // we treat completed shards as 0-level + return 0; + } + + // storing processing node to make sure we track progress and avoid circular dependencies + processedNodes.put(shardId, PROCESSING_NODE); + + int maxParentDepth = 0; + for (String parentId : shardInfo.getParentShardIds()) { + maxParentDepth = Math.max(maxParentDepth, + populateDepth(parentId, + shards, + processedNodes)); + } + + int currentNodeLevel = maxParentDepth + 1; + SortingNode previousValue = processedNodes.put(shardId, + new SortingNode(shardInfo, + currentNodeLevel)); + if (previousValue != PROCESSING_NODE) { + throw new IllegalStateException("Validation failed. Depth for shardId " + shardId + " was populated twice"); + } + + return currentNodeLevel; + } + + /** + * Class to store depth of shards during prioritization. + */ + private static class SortingNode implements + Comparable { + private final ShardInfo shardInfo; + private final int depth; + + public SortingNode(ShardInfo shardInfo, + int depth) { + this.shardInfo = shardInfo; + this.depth = depth; + } + + public int getDepth() { + return depth; + } + + @Override + public int compareTo(SortingNode o) { + return Integer.compare(depth, + o.depth); + } + } +} diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumer.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumer.java index 10dacc04..b6cc76aa 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumer.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumer.java @@ -297,36 +297,35 @@ class ShardConsumer { */ // CHECKSTYLE:OFF CyclomaticComplexity void updateState(boolean taskCompletedSuccessfully) { + if (currentState == ShardConsumerState.SHUTDOWN_COMPLETE) { + // Shutdown was completed and there nothing we can do after that + return; + } + if ((currentTask == null) && beginShutdown) { + // Shard didn't start any tasks and can be shutdown fast + currentState = ShardConsumerState.SHUTDOWN_COMPLETE; + return; + } + if (beginShutdown && currentState != ShardConsumerState.SHUTTING_DOWN) { + // Shard received signal to start shutdown. + // Whatever task we were working on should be stopped and shutdown task should be executed + currentState = ShardConsumerState.SHUTTING_DOWN; + return; + } switch (currentState) { case WAITING_ON_PARENT_SHARDS: if (taskCompletedSuccessfully && TaskType.BLOCK_ON_PARENT_SHARDS.equals(currentTask.getTaskType())) { - if (beginShutdown) { - currentState = ShardConsumerState.SHUTTING_DOWN; - } else { - currentState = ShardConsumerState.INITIALIZING; - } - } else if ((currentTask == null) && beginShutdown) { - currentState = ShardConsumerState.SHUTDOWN_COMPLETE; + currentState = ShardConsumerState.INITIALIZING; } break; case INITIALIZING: if (taskCompletedSuccessfully && TaskType.INITIALIZE.equals(currentTask.getTaskType())) { - if (beginShutdown) { - currentState = ShardConsumerState.SHUTTING_DOWN; - } else { - currentState = ShardConsumerState.PROCESSING; - } - } else if ((currentTask == null) && beginShutdown) { - currentState = ShardConsumerState.SHUTDOWN_COMPLETE; + currentState = ShardConsumerState.PROCESSING; } break; case PROCESSING: if (taskCompletedSuccessfully && TaskType.PROCESS.equals(currentTask.getTaskType())) { - if (beginShutdown) { - currentState = ShardConsumerState.SHUTTING_DOWN; - } else { - currentState = ShardConsumerState.PROCESSING; - } + currentState = ShardConsumerState.PROCESSING; } break; case SHUTTING_DOWN: @@ -335,8 +334,6 @@ class ShardConsumer { currentState = ShardConsumerState.SHUTDOWN_COMPLETE; } break; - case SHUTDOWN_COMPLETE: - break; default: LOG.error("Unexpected state: " + currentState); break; diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardInfo.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardInfo.java index 54f64568..9890d02f 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardInfo.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardInfo.java @@ -19,6 +19,8 @@ import java.util.Collections; import java.util.LinkedList; import java.util.List; +import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber; + /** * Used to pass shard related info among different classes and as a key to the map of shard consumers. */ @@ -28,13 +30,18 @@ class ShardInfo { private final String concurrencyToken; // Sorted list of parent shardIds. private final List parentShardIds; + private final ExtendedSequenceNumber checkpoint; /** * @param shardId Kinesis shardId * @param concurrencyToken Used to differentiate between lost and reclaimed leases * @param parentShardIds Parent shards of the shard identified by Kinesis shardId + * @param checkpoint the latest checkpoint from lease */ - public ShardInfo(String shardId, String concurrencyToken, Collection parentShardIds) { + public ShardInfo(String shardId, + String concurrencyToken, + Collection parentShardIds, + ExtendedSequenceNumber checkpoint) { this.shardId = shardId; this.concurrencyToken = concurrencyToken; this.parentShardIds = new LinkedList(); @@ -44,6 +51,7 @@ class ShardInfo { // ShardInfo stores parent shard Ids in canonical order in the parentShardIds list. // This makes it easy to check for equality in ShardInfo.equals method. Collections.sort(this.parentShardIds); + this.checkpoint = checkpoint; } /** @@ -67,6 +75,13 @@ class ShardInfo { return new LinkedList(parentShardIds); } + /** + * @return completion status of the shard + */ + protected boolean isCompleted() { + return ExtendedSequenceNumber.SHARD_END.equals(checkpoint); + } + /** * {@inheritDoc} */ @@ -77,6 +92,7 @@ class ShardInfo { result = prime * result + ((concurrencyToken == null) ? 0 : concurrencyToken.hashCode()); result = prime * result + ((parentShardIds == null) ? 0 : parentShardIds.hashCode()); result = prime * result + ((shardId == null) ? 0 : shardId.hashCode()); + result = prime * result + ((checkpoint == null) ? 0 : checkpoint.hashCode()); return result; } @@ -126,6 +142,13 @@ class ShardInfo { } else if (!shardId.equals(other.shardId)) { return false; } + if (checkpoint == null) { + if (other.checkpoint != null) { + return false; + } + } else if (!checkpoint.equals(other.checkpoint)) { + return false; + } return true; } @@ -135,7 +158,44 @@ class ShardInfo { @Override public String toString() { return "ShardInfo [shardId=" + shardId + ", concurrencyToken=" + concurrencyToken + ", parentShardIds=" - + parentShardIds + "]"; + + parentShardIds + ", checkpoint=" + checkpoint + "]"; + } + + /** + * Builder class for ShardInfo. + */ + public static class Builder { + private String shardId; + private String concurrencyToken; + private List parentShardIds = Collections.emptyList(); + private ExtendedSequenceNumber checkpoint = ExtendedSequenceNumber.LATEST; + + public Builder() { + } + + public Builder withShardId(String shardId) { + this.shardId = shardId; + return this; + } + + public Builder withConcurrencyToken(String concurrencyToken) { + this.concurrencyToken = concurrencyToken; + return this; + } + + public Builder withParentShards(List parentShardIds) { + this.parentShardIds = parentShardIds; + return this; + } + + public Builder withCheckpoint(ExtendedSequenceNumber checkpoint) { + this.checkpoint = checkpoint; + return this; + } + + public ShardInfo build() { + return new ShardInfo(shardId, concurrencyToken, parentShardIds, checkpoint); + } } } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardPrioritization.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardPrioritization.java new file mode 100644 index 00000000..54f7517d --- /dev/null +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardPrioritization.java @@ -0,0 +1,19 @@ +package com.amazonaws.services.kinesis.clientlibrary.lib.worker; + +import java.util.List; + +/** + * Provides logic to prioritize or filter shards before their execution. + */ +public interface ShardPrioritization { + + /** + * Returns new list of shards ordered based on their priority. + * Resulted list may have fewer shards compared to original list + * + * @param original + * list of shards needed to be prioritized + * @return new list that contains only shards that should be processed + */ + List prioritize(List original); +} diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java index 50861c29..3da1f2cd 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java @@ -38,6 +38,7 @@ import com.amazonaws.services.kinesis.AmazonKinesisClient; import com.amazonaws.services.kinesis.clientlibrary.interfaces.ICheckpoint; import com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessor; import com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessorFactory; +import com.amazonaws.services.kinesis.clientlibrary.lib.worker.Worker.Builder; import com.amazonaws.services.kinesis.clientlibrary.proxies.KinesisProxyFactory; import com.amazonaws.services.kinesis.clientlibrary.types.ShutdownReason; import com.amazonaws.services.kinesis.leases.exceptions.LeasingException; @@ -80,6 +81,8 @@ public class Worker implements Runnable { private final KinesisClientLibLeaseCoordinator leaseCoordinator; private final ShardSyncTaskManager controlServer; + private final ShardPrioritization shardPrioritization; + private volatile boolean shutdown; private volatile long shutdownStartTimeMillis; @@ -231,7 +234,8 @@ public class Worker implements Runnable { execService, metricsFactory, config.getTaskBackoffTimeMillis(), - config.getFailoverTimeMillis()); + config.getFailoverTimeMillis(), + config.getShardPrioritizationStrategy()); // If a region name was explicitly specified, use it as the region for Amazon Kinesis and Amazon DynamoDB. if (config.getRegionName() != null) { Region region = RegionUtils.getRegion(config.getRegionName()); @@ -271,6 +275,7 @@ public class Worker implements Runnable { * consumption) * @param metricsFactory Metrics factory used to emit metrics * @param taskBackoffTimeMillis Backoff period when tasks encounter an exception + * @param shardPrioritization Provides prioritization logic to decide which available shards process first */ // NOTE: This has package level access solely for testing // CHECKSTYLE:IGNORE ParameterNumber FOR NEXT 10 LINES @@ -286,7 +291,8 @@ public class Worker implements Runnable { ExecutorService execService, IMetricsFactory metricsFactory, long taskBackoffTimeMillis, - long failoverTimeMillis) { + long failoverTimeMillis, + ShardPrioritization shardPrioritization) { this.applicationName = applicationName; this.recordProcessorFactory = recordProcessorFactory; this.streamConfig = streamConfig; @@ -308,6 +314,7 @@ public class Worker implements Runnable { executorService); this.taskBackoffTimeMillis = taskBackoffTimeMillis; this.failoverTimeMillis = failoverTimeMillis; + this.shardPrioritization = shardPrioritization; } /** @@ -449,12 +456,13 @@ public class Worker implements Runnable { private List getShardInfoForAssignments() { List assignedStreamShards = leaseCoordinator.getCurrentAssignments(); + List prioritizedShards = shardPrioritization.prioritize(assignedStreamShards); - if ((assignedStreamShards != null) && (!assignedStreamShards.isEmpty())) { + if ((prioritizedShards != null) && (!prioritizedShards.isEmpty())) { if (wlog.isInfoEnabled()) { StringBuilder builder = new StringBuilder(); boolean firstItem = true; - for (ShardInfo shardInfo : assignedStreamShards) { + for (ShardInfo shardInfo : prioritizedShards) { if (!firstItem) { builder.append(", "); } @@ -467,7 +475,7 @@ public class Worker implements Runnable { wlog.info("No activities assigned"); } - return assignedStreamShards; + return prioritizedShards; } /** @@ -780,6 +788,7 @@ public class Worker implements Runnable { private AmazonCloudWatch cloudWatchClient; private IMetricsFactory metricsFactory; private ExecutorService execService; + private ShardPrioritization shardPrioritization; /** * Default constructor. @@ -879,6 +888,19 @@ public class Worker implements Runnable { return this; } + /** + * Provides logic how to prioritize shard processing. + * + * @param shardPrioritization + * shardPrioritization is responsible to order shards before processing + * + * @return A reference to this updated object so that method calls can be chained together. + */ + public Builder shardPrioritization(ShardPrioritization shardPrioritization) { + this.shardPrioritization = shardPrioritization; + return this; + } + /** * Build the Worker instance. * @@ -937,6 +959,9 @@ public class Worker implements Runnable { if (metricsFactory == null) { metricsFactory = getMetricsFactory(cloudWatchClient, config); } + if (shardPrioritization == null) { + shardPrioritization = new ParentsFirstShardPrioritization(1); + } return new Worker(config.getApplicationName(), recordProcessorFactory, @@ -965,7 +990,8 @@ public class Worker implements Runnable { execService, metricsFactory, config.getTaskBackoffTimeMillis(), - config.getFailoverTimeMillis()); + config.getFailoverTimeMillis(), + shardPrioritization); } } diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/BlockOnParentShardTaskTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/BlockOnParentShardTaskTest.java index 4c98701a..a42e0683 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/BlockOnParentShardTaskTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/BlockOnParentShardTaskTest.java @@ -20,12 +20,11 @@ import static org.mockito.Mockito.when; import java.util.ArrayList; import java.util.List; -import junit.framework.Assert; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.junit.After; import org.junit.AfterClass; +import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -47,7 +46,7 @@ public class BlockOnParentShardTaskTest { private final String shardId = "shardId-97"; private final String concurrencyToken = "testToken"; private final List emptyParentShardIds = new ArrayList(); - ShardInfo defaultShardInfo = new ShardInfo(shardId, concurrencyToken, emptyParentShardIds); + ShardInfo defaultShardInfo = new ShardInfo(shardId, concurrencyToken, emptyParentShardIds, ExtendedSequenceNumber.TRIM_HORIZON); /** * @throws java.lang.Exception @@ -122,14 +121,14 @@ public class BlockOnParentShardTaskTest { // test single parent parentShardIds.add(parent1ShardId); - shardInfo = new ShardInfo(shardId, concurrencyToken, parentShardIds); + shardInfo = new ShardInfo(shardId, concurrencyToken, parentShardIds, ExtendedSequenceNumber.TRIM_HORIZON); task = new BlockOnParentShardTask(shardInfo, leaseManager, backoffTimeInMillis); result = task.call(); Assert.assertNull(result.getException()); // test two parents parentShardIds.add(parent2ShardId); - shardInfo = new ShardInfo(shardId, concurrencyToken, parentShardIds); + shardInfo = new ShardInfo(shardId, concurrencyToken, parentShardIds, ExtendedSequenceNumber.TRIM_HORIZON); task = new BlockOnParentShardTask(shardInfo, leaseManager, backoffTimeInMillis); result = task.call(); Assert.assertNull(result.getException()); @@ -164,14 +163,14 @@ public class BlockOnParentShardTaskTest { // test single parent parentShardIds.add(parent1ShardId); - shardInfo = new ShardInfo(shardId, concurrencyToken, parentShardIds); + shardInfo = new ShardInfo(shardId, concurrencyToken, parentShardIds, ExtendedSequenceNumber.TRIM_HORIZON); task = new BlockOnParentShardTask(shardInfo, leaseManager, backoffTimeInMillis); result = task.call(); Assert.assertNotNull(result.getException()); // test two parents parentShardIds.add(parent2ShardId); - shardInfo = new ShardInfo(shardId, concurrencyToken, parentShardIds); + shardInfo = new ShardInfo(shardId, concurrencyToken, parentShardIds, ExtendedSequenceNumber.TRIM_HORIZON); task = new BlockOnParentShardTask(shardInfo, leaseManager, backoffTimeInMillis); result = task.call(); Assert.assertNotNull(result.getException()); @@ -191,7 +190,7 @@ public class BlockOnParentShardTaskTest { String parentShardId = "shardId-1"; List parentShardIds = new ArrayList<>(); parentShardIds.add(parentShardId); - ShardInfo shardInfo = new ShardInfo(shardId, concurrencyToken, parentShardIds); + ShardInfo shardInfo = new ShardInfo(shardId, concurrencyToken, parentShardIds, ExtendedSequenceNumber.TRIM_HORIZON); TaskResult result = null; KinesisClientLease parentLease = new KinesisClientLease(); ILeaseManager leaseManager = mock(ILeaseManager.class); diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisDataFetcherTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisDataFetcherTest.java index 556a1e0e..dd56a256 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisDataFetcherTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisDataFetcherTest.java @@ -48,7 +48,7 @@ public class KinesisDataFetcherTest { private static final int MAX_RECORDS = 1; private static final String SHARD_ID = "shardId-1"; private static final String AT_SEQUENCE_NUMBER = ShardIteratorType.AT_SEQUENCE_NUMBER.toString(); - private static final ShardInfo SHARD_INFO = new ShardInfo(SHARD_ID, null, null); + private static final ShardInfo SHARD_INFO = new ShardInfo(SHARD_ID, null, null, null); private static final InitialPositionInStreamExtended INITIAL_POSITION_LATEST = InitialPositionInStreamExtended.newInitialPosition(InitialPositionInStream.LATEST); private static final InitialPositionInStreamExtended INITIAL_POSITION_TRIM_HORIZON = diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ParentsFirstShardPrioritizationUnitTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ParentsFirstShardPrioritizationUnitTest.java new file mode 100644 index 00000000..35b56b32 --- /dev/null +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ParentsFirstShardPrioritizationUnitTest.java @@ -0,0 +1,162 @@ +package com.amazonaws.services.kinesis.clientlibrary.lib.worker; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Random; + +import org.junit.Test; + +public class ParentsFirstShardPrioritizationUnitTest { + + @Test(expected = IllegalArgumentException.class) + public void testMaxDepthNegativeShouldFail() { + new ParentsFirstShardPrioritization(-1); + } + + @Test(expected = IllegalArgumentException.class) + public void testMaxDepthZeroShouldFail() { + new ParentsFirstShardPrioritization(0); + } + + @Test + public void testMaxDepthPositiveShouldNotFail() { + new ParentsFirstShardPrioritization(1); + } + + @Test + public void testSorting() { + Random random = new Random(987654); + int numberOfShards = 7; + + List shardIdsDependencies = new ArrayList<>(); + shardIdsDependencies.add("unknown"); + List original = new ArrayList<>(); + for (int shardNumber = 0; shardNumber < numberOfShards; shardNumber++) { + String shardId = shardId(shardNumber); + original.add(shardInfo(shardId, shardIdsDependencies)); + shardIdsDependencies.add(shardId); + } + + ParentsFirstShardPrioritization ordering = new ParentsFirstShardPrioritization(Integer.MAX_VALUE); + + // shuffle original list as it is already ordered in right way + Collections.shuffle(original, random); + List ordered = ordering.prioritize(original); + + assertEquals(numberOfShards, ordered.size()); + for (int shardNumber = 0; shardNumber < numberOfShards; shardNumber++) { + String shardId = shardId(shardNumber); + assertEquals(shardId, ordered.get(shardNumber).getShardId()); + } + } + + @Test + public void testSortingAndFiltering() { + Random random = new Random(45677); + int numberOfShards = 10; + + List shardIdsDependencies = new ArrayList<>(); + shardIdsDependencies.add("unknown"); + List original = new ArrayList<>(); + for (int shardNumber = 0; shardNumber < numberOfShards; shardNumber++) { + String shardId = shardId(shardNumber); + original.add(shardInfo(shardId, shardIdsDependencies)); + shardIdsDependencies.add(shardId); + } + + int maxDepth = 3; + ParentsFirstShardPrioritization ordering = new ParentsFirstShardPrioritization(maxDepth); + + // shuffle original list as it is already ordered in right way + Collections.shuffle(original, random); + List ordered = ordering.prioritize(original); + // in this case every shard has its own level, so we don't expect to + // have more shards than max depth + assertEquals(maxDepth, ordered.size()); + + for (int shardNumber = 0; shardNumber < maxDepth; shardNumber++) { + String shardId = shardId(shardNumber); + assertEquals(shardId, ordered.get(shardNumber).getShardId()); + } + } + + @Test + public void testSimpleOrdering() { + Random random = new Random(1234); + int numberOfShards = 10; + + String parentId = "unknown"; + List original = new ArrayList<>(); + for (int shardNumber = 0; shardNumber < numberOfShards; shardNumber++) { + String shardId = shardId(shardNumber); + original.add(shardInfo(shardId, parentId)); + parentId = shardId; + } + + ParentsFirstShardPrioritization ordering = new ParentsFirstShardPrioritization(Integer.MAX_VALUE); + + // shuffle original list as it is already ordered in right way + Collections.shuffle(original, random); + List ordered = ordering.prioritize(original); + assertEquals(numberOfShards, ordered.size()); + for (int shardNumber = 0; shardNumber < numberOfShards; shardNumber++) { + String shardId = shardId(shardNumber); + assertEquals(shardId, ordered.get(shardNumber).getShardId()); + } + } + + /** + * This should be impossible as shards don't have circular dependencies, + * but this code should handle it properly and fail + */ + @Test + public void testCircularDependencyBetweenShards() { + Random random = new Random(13468798); + int numberOfShards = 10; + + // shard-0 will point in middle shard (shard-5) in current test + String parentId = shardId(numberOfShards / 2); + List original = new ArrayList<>(); + for (int shardNumber = 0; shardNumber < numberOfShards; shardNumber++) { + String shardId = shardId(shardNumber); + original.add(shardInfo(shardId, parentId)); + parentId = shardId; + } + + ParentsFirstShardPrioritization ordering = new ParentsFirstShardPrioritization(Integer.MAX_VALUE); + + // shuffle original list as it is already ordered in right way + Collections.shuffle(original, random); + try { + ordering.prioritize(original); + fail("Processing should fail in case we have circular dependency"); + } catch (IllegalArgumentException expected) { + + } + } + + private String shardId(int shardNumber) { + return "shardId-" + shardNumber; + } + + private static ShardInfo shardInfo(String shardId, List parentShardIds) { + // copy into new list just in case ShardInfo will stop doing it + List newParentShardIds = new ArrayList<>(parentShardIds); + return new ShardInfo.Builder() + .withShardId(shardId) + .withParentShards(newParentShardIds) + .build(); + } + + private static ShardInfo shardInfo(String shardId, String... parentShardIds) { + return new ShardInfo.Builder() + .withShardId(shardId) + .withParentShards(Arrays.asList(parentShardIds)) + .build(); + } +} diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ProcessTaskTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ProcessTaskTest.java index 6576e47f..f1d908f0 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ProcessTaskTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ProcessTaskTest.java @@ -87,7 +87,7 @@ public class ProcessTaskTest { new StreamConfig(null, maxRecords, idleTimeMillis, callProcessRecordsForEmptyRecordList, skipCheckpointValidationValue, INITIAL_POSITION_LATEST); - final ShardInfo shardInfo = new ShardInfo(shardId, null, null); + final ShardInfo shardInfo = new ShardInfo(shardId, null, null, null); processTask = new ProcessTask( shardInfo, config, mockRecordProcessor, mockCheckpointer, mockDataFetcher, taskBackoffTimeMillis); } diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/RecordProcessorCheckpointerTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/RecordProcessorCheckpointerTest.java index 4741ea14..d5f6b53f 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/RecordProcessorCheckpointerTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/RecordProcessorCheckpointerTest.java @@ -75,7 +75,7 @@ public class RecordProcessorCheckpointerTest { */ @Test public final void testCheckpoint() throws Exception { - ShardInfo shardInfo = new ShardInfo(shardId, testConcurrencyToken, null); + ShardInfo shardInfo = new ShardInfo(shardId, testConcurrencyToken, null, ExtendedSequenceNumber.TRIM_HORIZON); // First call to checkpoint RecordProcessorCheckpointer processingCheckpointer = @@ -98,7 +98,7 @@ public class RecordProcessorCheckpointerTest { */ @Test public final void testCheckpointRecord() throws Exception { - ShardInfo shardInfo = new ShardInfo(shardId, testConcurrencyToken, null); + ShardInfo shardInfo = new ShardInfo(shardId, testConcurrencyToken, null, ExtendedSequenceNumber.TRIM_HORIZON); SequenceNumberValidator sequenceNumberValidator = new SequenceNumberValidator(null, shardId, false); RecordProcessorCheckpointer processingCheckpointer = @@ -117,7 +117,7 @@ public class RecordProcessorCheckpointerTest { */ @Test public final void testCheckpointSubRecord() throws Exception { - ShardInfo shardInfo = new ShardInfo(shardId, testConcurrencyToken, null); + ShardInfo shardInfo = new ShardInfo(shardId, testConcurrencyToken, null, ExtendedSequenceNumber.TRIM_HORIZON); SequenceNumberValidator sequenceNumberValidator = new SequenceNumberValidator(null, shardId, false); RecordProcessorCheckpointer processingCheckpointer = @@ -137,7 +137,7 @@ public class RecordProcessorCheckpointerTest { */ @Test public final void testCheckpointSequenceNumber() throws Exception { - ShardInfo shardInfo = new ShardInfo(shardId, testConcurrencyToken, null); + ShardInfo shardInfo = new ShardInfo(shardId, testConcurrencyToken, null, ExtendedSequenceNumber.TRIM_HORIZON); SequenceNumberValidator sequenceNumberValidator = new SequenceNumberValidator(null, shardId, false); RecordProcessorCheckpointer processingCheckpointer = @@ -155,7 +155,7 @@ public class RecordProcessorCheckpointerTest { */ @Test public final void testCheckpointExtendedSequenceNumber() throws Exception { - ShardInfo shardInfo = new ShardInfo(shardId, testConcurrencyToken, null); + ShardInfo shardInfo = new ShardInfo(shardId, testConcurrencyToken, null, ExtendedSequenceNumber.TRIM_HORIZON); SequenceNumberValidator sequenceNumberValidator = new SequenceNumberValidator(null, shardId, false); RecordProcessorCheckpointer processingCheckpointer = @@ -173,7 +173,7 @@ public class RecordProcessorCheckpointerTest { */ @Test public final void testUpdate() throws Exception { - ShardInfo shardInfo = new ShardInfo(shardId, testConcurrencyToken, null); + ShardInfo shardInfo = new ShardInfo(shardId, testConcurrencyToken, null, ExtendedSequenceNumber.TRIM_HORIZON); RecordProcessorCheckpointer checkpointer = new RecordProcessorCheckpointer(shardInfo, checkpoint, null); @@ -193,7 +193,7 @@ public class RecordProcessorCheckpointerTest { */ @Test public final void testClientSpecifiedCheckpoint() throws Exception { - ShardInfo shardInfo = new ShardInfo(shardId, testConcurrencyToken, null); + ShardInfo shardInfo = new ShardInfo(shardId, testConcurrencyToken, null, ExtendedSequenceNumber.TRIM_HORIZON); SequenceNumberValidator validator = mock(SequenceNumberValidator.class); Mockito.doNothing().when(validator).validateSequenceNumber(anyString()); @@ -290,7 +290,7 @@ public class RecordProcessorCheckpointerTest { @SuppressWarnings("serial") @Test public final void testMixedCheckpointCalls() throws Exception { - ShardInfo shardInfo = new ShardInfo(shardId, testConcurrencyToken, null); + ShardInfo shardInfo = new ShardInfo(shardId, testConcurrencyToken, null, ExtendedSequenceNumber.TRIM_HORIZON); SequenceNumberValidator validator = mock(SequenceNumberValidator.class); Mockito.doNothing().when(validator).validateSequenceNumber(anyString()); diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumerTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumerTest.java index d8d39377..26337381 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumerTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumerTest.java @@ -92,7 +92,7 @@ public class ShardConsumerTest { @SuppressWarnings("unchecked") @Test public final void testInitializationStateUponFailure() throws Exception { - ShardInfo shardInfo = new ShardInfo("s-0-0", "testToken", null); + ShardInfo shardInfo = new ShardInfo("s-0-0", "testToken", null, ExtendedSequenceNumber.TRIM_HORIZON); ICheckpoint checkpoint = mock(ICheckpoint.class); when(checkpoint.getCheckpoint(anyString())).thenThrow(NullPointerException.class); @@ -141,7 +141,7 @@ public class ShardConsumerTest { @SuppressWarnings("unchecked") @Test public final void testInitializationStateUponSubmissionFailure() throws Exception { - ShardInfo shardInfo = new ShardInfo("s-0-0", "testToken", null); + ShardInfo shardInfo = new ShardInfo("s-0-0", "testToken", null, ExtendedSequenceNumber.TRIM_HORIZON); ICheckpoint checkpoint = mock(ICheckpoint.class); ExecutorService spyExecutorService = spy(executorService); @@ -189,7 +189,7 @@ public class ShardConsumerTest { @SuppressWarnings("unchecked") @Test public final void testRecordProcessorThrowable() throws Exception { - ShardInfo shardInfo = new ShardInfo("s-0-0", "testToken", null); + ShardInfo shardInfo = new ShardInfo("s-0-0", "testToken", null, ExtendedSequenceNumber.TRIM_HORIZON); ICheckpoint checkpoint = mock(ICheckpoint.class); IRecordProcessor processor = mock(IRecordProcessor.class); IKinesisProxy streamProxy = mock(IKinesisProxy.class); @@ -289,7 +289,7 @@ public class ShardConsumerTest { callProcessRecordsForEmptyRecordList, skipCheckpointValidationValue, INITIAL_POSITION_LATEST); - ShardInfo shardInfo = new ShardInfo(streamShardId, testConcurrencyToken, null); + ShardInfo shardInfo = new ShardInfo(streamShardId, testConcurrencyToken, null, null); ShardConsumer consumer = new ShardConsumer(shardInfo, streamConfig, @@ -379,7 +379,7 @@ public class ShardConsumerTest { skipCheckpointValidationValue, atTimestamp); - ShardInfo shardInfo = new ShardInfo(streamShardId, testConcurrencyToken, null); + ShardInfo shardInfo = new ShardInfo(streamShardId, testConcurrencyToken, null, ExtendedSequenceNumber.TRIM_HORIZON); ShardConsumer consumer = new ShardConsumer(shardInfo, streamConfig, diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardInfoTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardInfoTest.java index d62d880d..a2434d69 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardInfoTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardInfoTest.java @@ -20,11 +20,12 @@ import java.util.List; import java.util.Set; import java.util.UUID; -import junit.framework.Assert; - +import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber; + public class ShardInfoTest { private static final String CONCURRENCY_TOKEN = UUID.randomUUID().toString(); private static final String SHARD_ID = "shardId-test"; @@ -37,12 +38,12 @@ public class ShardInfoTest { parentShardIds.add("shard-1"); parentShardIds.add("shard-2"); - testShardInfo = new ShardInfo(SHARD_ID, CONCURRENCY_TOKEN, parentShardIds); + testShardInfo = new ShardInfo(SHARD_ID, CONCURRENCY_TOKEN, parentShardIds, ExtendedSequenceNumber.LATEST); } @Test public void testPacboyShardInfoEqualsWithSameArgs() { - ShardInfo equalShardInfo = new ShardInfo(SHARD_ID, CONCURRENCY_TOKEN, parentShardIds); + ShardInfo equalShardInfo = new ShardInfo(SHARD_ID, CONCURRENCY_TOKEN, parentShardIds, ExtendedSequenceNumber.LATEST); Assert.assertTrue("Equal should return true for arguments all the same", testShardInfo.equals(equalShardInfo)); } @@ -53,18 +54,18 @@ public class ShardInfoTest { @Test public void testPacboyShardInfoEqualsForShardId() { - ShardInfo diffShardInfo = new ShardInfo("shardId-diff", CONCURRENCY_TOKEN, parentShardIds); + ShardInfo diffShardInfo = new ShardInfo("shardId-diff", CONCURRENCY_TOKEN, parentShardIds, ExtendedSequenceNumber.LATEST); Assert.assertFalse("Equal should return false with different shard id", diffShardInfo.equals(testShardInfo)); - diffShardInfo = new ShardInfo(null, CONCURRENCY_TOKEN, parentShardIds); + diffShardInfo = new ShardInfo(null, CONCURRENCY_TOKEN, parentShardIds, ExtendedSequenceNumber.LATEST); Assert.assertFalse("Equal should return false with null shard id", diffShardInfo.equals(testShardInfo)); } @Test public void testPacboyShardInfoEqualsForfToken() { - ShardInfo diffShardInfo = new ShardInfo(SHARD_ID, UUID.randomUUID().toString(), parentShardIds); + ShardInfo diffShardInfo = new ShardInfo(SHARD_ID, UUID.randomUUID().toString(), parentShardIds, ExtendedSequenceNumber.LATEST); Assert.assertFalse("Equal should return false with different concurrency token", diffShardInfo.equals(testShardInfo)); - diffShardInfo = new ShardInfo(SHARD_ID, null, parentShardIds); + diffShardInfo = new ShardInfo(SHARD_ID, null, parentShardIds, ExtendedSequenceNumber.LATEST); Assert.assertFalse("Equal should return false for null concurrency token", diffShardInfo.equals(testShardInfo)); } @@ -74,7 +75,7 @@ public class ShardInfoTest { differentlyOrderedParentShardIds.add("shard-2"); differentlyOrderedParentShardIds.add("shard-1"); ShardInfo shardInfoWithDifferentlyOrderedParentShardIds = - new ShardInfo(SHARD_ID, CONCURRENCY_TOKEN, differentlyOrderedParentShardIds); + new ShardInfo(SHARD_ID, CONCURRENCY_TOKEN, differentlyOrderedParentShardIds, ExtendedSequenceNumber.LATEST); Assert.assertTrue("Equal should return true even with parent shard Ids reordered", shardInfoWithDifferentlyOrderedParentShardIds.equals(testShardInfo)); } @@ -84,16 +85,24 @@ public class ShardInfoTest { Set diffParentIds = new HashSet<>(); diffParentIds.add("shard-3"); diffParentIds.add("shard-4"); - ShardInfo diffShardInfo = new ShardInfo(SHARD_ID, CONCURRENCY_TOKEN, diffParentIds); + ShardInfo diffShardInfo = new ShardInfo(SHARD_ID, CONCURRENCY_TOKEN, diffParentIds, ExtendedSequenceNumber.LATEST); Assert.assertFalse("Equal should return false with different parent shard Ids", diffShardInfo.equals(testShardInfo)); - diffShardInfo = new ShardInfo(SHARD_ID, CONCURRENCY_TOKEN, null); + diffShardInfo = new ShardInfo(SHARD_ID, CONCURRENCY_TOKEN, null, ExtendedSequenceNumber.LATEST); Assert.assertFalse("Equal should return false with null parent shard Ids", diffShardInfo.equals(testShardInfo)); } + @Test + public void testPacboyShardInfoEqualsForCheckpoint() { + ShardInfo diffShardInfo = new ShardInfo(SHARD_ID, CONCURRENCY_TOKEN, parentShardIds, ExtendedSequenceNumber.SHARD_END); + Assert.assertFalse("Equal should return false with different checkpoint", diffShardInfo.equals(testShardInfo)); + diffShardInfo = new ShardInfo(SHARD_ID, CONCURRENCY_TOKEN, parentShardIds, null); + Assert.assertFalse("Equal should return false with null checkpoint", diffShardInfo.equals(testShardInfo)); + } + @Test public void testPacboyShardInfoSameHashCode() { - ShardInfo equalShardInfo = new ShardInfo(SHARD_ID, CONCURRENCY_TOKEN, parentShardIds); + ShardInfo equalShardInfo = new ShardInfo(SHARD_ID, CONCURRENCY_TOKEN, parentShardIds, ExtendedSequenceNumber.LATEST); Assert.assertTrue("Shard info objects should have same hashCode for the same arguments", equalShardInfo.hashCode() == testShardInfo.hashCode()); } diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTaskTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTaskTest.java index a2302ad0..67b42a0e 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTaskTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTaskTest.java @@ -20,10 +20,9 @@ import static org.mockito.Mockito.when; import java.util.HashSet; import java.util.Set; -import junit.framework.Assert; - import org.junit.After; import org.junit.AfterClass; +import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -50,7 +49,8 @@ public class ShutdownTaskTest { String defaultShardId = "shardId-0000397840"; ShardInfo defaultShardInfo = new ShardInfo(defaultShardId, defaultConcurrencyToken, - defaultParentShardIds); + defaultParentShardIds, + ExtendedSequenceNumber.LATEST); IRecordProcessor defaultRecordProcessor = new TestStreamlet(); /** diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/WorkerTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/WorkerTest.java index f0b42671..a68c229b 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/WorkerTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/WorkerTest.java @@ -105,6 +105,7 @@ public class WorkerTest { InitialPositionInStreamExtended.newInitialPosition(InitialPositionInStream.LATEST); private static final InitialPositionInStreamExtended INITIAL_POSITION_TRIM_HORIZON = InitialPositionInStreamExtended.newInitialPosition(InitialPositionInStream.TRIM_HORIZON); + private final ShardPrioritization shardPrioritization = new NoOpShardPrioritization(); // CHECKSTYLE:IGNORE AnonInnerLengthCheck FOR NEXT 50 LINES private static final com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorFactory SAMPLE_RECORD_PROCESSOR_FACTORY = @@ -192,14 +193,15 @@ public class WorkerTest { execService, nullMetricsFactory, taskBackoffTimeMillis, - failoverTimeMillis); - ShardInfo shardInfo = new ShardInfo(dummyKinesisShardId, testConcurrencyToken, null); + failoverTimeMillis, + shardPrioritization); + ShardInfo shardInfo = new ShardInfo(dummyKinesisShardId, testConcurrencyToken, null, ExtendedSequenceNumber.TRIM_HORIZON); ShardConsumer consumer = worker.createOrGetShardConsumer(shardInfo, streamletFactory); Assert.assertNotNull(consumer); ShardConsumer consumer2 = worker.createOrGetShardConsumer(shardInfo, streamletFactory); Assert.assertSame(consumer, consumer2); ShardInfo shardInfoWithSameShardIdButDifferentConcurrencyToken = - new ShardInfo(dummyKinesisShardId, anotherConcurrencyToken, null); + new ShardInfo(dummyKinesisShardId, anotherConcurrencyToken, null, ExtendedSequenceNumber.TRIM_HORIZON); ShardConsumer consumer3 = worker.createOrGetShardConsumer(shardInfoWithSameShardIdButDifferentConcurrencyToken, streamletFactory); Assert.assertNotNull(consumer3); @@ -241,12 +243,13 @@ public class WorkerTest { execService, nullMetricsFactory, taskBackoffTimeMillis, - failoverTimeMillis); + failoverTimeMillis, + shardPrioritization); - ShardInfo shardInfo1 = new ShardInfo(dummyKinesisShardId, concurrencyToken, null); + ShardInfo shardInfo1 = new ShardInfo(dummyKinesisShardId, concurrencyToken, null, ExtendedSequenceNumber.TRIM_HORIZON); ShardInfo duplicateOfShardInfo1ButWithAnotherConcurrencyToken = - new ShardInfo(dummyKinesisShardId, anotherConcurrencyToken, null); - ShardInfo shardInfo2 = new ShardInfo(anotherDummyKinesisShardId, concurrencyToken, null); + new ShardInfo(dummyKinesisShardId, anotherConcurrencyToken, null, ExtendedSequenceNumber.TRIM_HORIZON); + ShardInfo shardInfo2 = new ShardInfo(anotherDummyKinesisShardId, concurrencyToken, null, ExtendedSequenceNumber.TRIM_HORIZON); ShardConsumer consumerOfShardInfo1 = worker.createOrGetShardConsumer(shardInfo1, streamletFactory); ShardConsumer consumerOfDuplicateOfShardInfo1ButWithAnotherConcurrencyToken = @@ -297,7 +300,8 @@ public class WorkerTest { execService, nullMetricsFactory, taskBackoffTimeMillis, - failoverTimeMillis); + failoverTimeMillis, + shardPrioritization); worker.run(); Assert.assertTrue(count > 0); } @@ -745,7 +749,8 @@ public class WorkerTest { executorService, metricsFactory, taskBackoffTimeMillis, - failoverTimeMillis); + failoverTimeMillis, + shardPrioritization); WorkerThread workerThread = new WorkerThread(worker); workerThread.start(); From 6ea3c0f8ddef8ad0f99b1a9d3e787dc051e87ff8 Mon Sep 17 00:00:00 2001 From: "Pfifer, Justin" Date: Wed, 17 Aug 2016 10:56:27 -0700 Subject: [PATCH 6/9] Don't Use Checkpoint for Equality in ShardInfo Don't include checkpoint in hashCode/equality for ShardInfo, since it changes. Checkpointing would cause the Worker to recreate the ShardConsumer. Add unit tests that verify the equality constraints. Remove the equality test for checkpoint from ShardInfoTests. Nothing appears to rely on the checkpoint being part of ShardInfo. Fix WorkerTest broken in overzealous simplification. --- .../clientlibrary/lib/worker/ShardInfo.java | 97 ++--------- .../clientlibrary/lib/worker/Worker.java | 96 +++++------ ...rentsFirstShardPrioritizationUnitTest.java | 43 ++++- .../lib/worker/ShardInfoTest.java | 23 ++- .../clientlibrary/lib/worker/WorkerTest.java | 157 +++++++++++++----- 5 files changed, 240 insertions(+), 176 deletions(-) diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardInfo.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardInfo.java index 9890d02f..f04a86ba 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardInfo.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardInfo.java @@ -19,13 +19,17 @@ import java.util.Collections; import java.util.LinkedList; import java.util.List; +import org.apache.commons.lang.builder.EqualsBuilder; + import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber; +import org.apache.commons.lang.builder.HashCodeBuilder; /** * Used to pass shard related info among different classes and as a key to the map of shard consumers. */ class ShardInfo { + private final String shardId; private final String concurrencyToken; // Sorted list of parent shardIds. @@ -33,10 +37,16 @@ class ShardInfo { private final ExtendedSequenceNumber checkpoint; /** - * @param shardId Kinesis shardId - * @param concurrencyToken Used to differentiate between lost and reclaimed leases - * @param parentShardIds Parent shards of the shard identified by Kinesis shardId - * @param checkpoint the latest checkpoint from lease + * Creates a new ShardInfo object. The checkpoint is not part of the equality, but is used for debugging output. + * + * @param shardId + * Kinesis shardId + * @param concurrencyToken + * Used to differentiate between lost and reclaimed leases + * @param parentShardIds + * Parent shards of the shard identified by Kinesis shardId + * @param checkpoint + * the latest checkpoint from lease */ public ShardInfo(String shardId, String concurrencyToken, @@ -87,20 +97,12 @@ class ShardInfo { */ @Override public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((concurrencyToken == null) ? 0 : concurrencyToken.hashCode()); - result = prime * result + ((parentShardIds == null) ? 0 : parentShardIds.hashCode()); - result = prime * result + ((shardId == null) ? 0 : shardId.hashCode()); - result = prime * result + ((checkpoint == null) ? 0 : checkpoint.hashCode()); - return result; + return new HashCodeBuilder().append(concurrencyToken).append(parentShardIds).append(shardId).toHashCode(); } /** * {@inheritDoc} */ - // CHECKSTYLE:OFF CyclomaticComplexity - // CHECKSTYLE:OFF NPathComplexity /** * This method assumes parentShardIds is ordered. The Worker.cleanupShardConsumers() method relies on this method * returning true for ShardInfo objects which may have been instantiated with parentShardIds in a different order @@ -121,39 +123,11 @@ class ShardInfo { return false; } ShardInfo other = (ShardInfo) obj; - if (concurrencyToken == null) { - if (other.concurrencyToken != null) { - return false; - } - } else if (!concurrencyToken.equals(other.concurrencyToken)) { - return false; - } - if (parentShardIds == null) { - if (other.parentShardIds != null) { - return false; - } - } else if (!parentShardIds.equals(other.parentShardIds)) { - return false; - } - if (shardId == null) { - if (other.shardId != null) { - return false; - } - } else if (!shardId.equals(other.shardId)) { - return false; - } - if (checkpoint == null) { - if (other.checkpoint != null) { - return false; - } - } else if (!checkpoint.equals(other.checkpoint)) { - return false; - } - return true; + return new EqualsBuilder().append(concurrencyToken, other.concurrencyToken) + .append(parentShardIds, other.parentShardIds).append(shardId, other.shardId).isEquals(); + } - // CHECKSTYLE:ON CyclomaticComplexity - // CHECKSTYLE:ON NPathComplexity @Override public String toString() { @@ -161,41 +135,6 @@ class ShardInfo { + parentShardIds + ", checkpoint=" + checkpoint + "]"; } - /** - * Builder class for ShardInfo. - */ - public static class Builder { - private String shardId; - private String concurrencyToken; - private List parentShardIds = Collections.emptyList(); - private ExtendedSequenceNumber checkpoint = ExtendedSequenceNumber.LATEST; - public Builder() { - } - - public Builder withShardId(String shardId) { - this.shardId = shardId; - return this; - } - - public Builder withConcurrencyToken(String concurrencyToken) { - this.concurrencyToken = concurrencyToken; - return this; - } - - public Builder withParentShards(List parentShardIds) { - this.parentShardIds = parentShardIds; - return this; - } - - public Builder withCheckpoint(ExtendedSequenceNumber checkpoint) { - this.checkpoint = checkpoint; - return this; - } - - public ShardInfo build() { - return new ShardInfo(shardId, concurrencyToken, parentShardIds, checkpoint); - } - } } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java index 3da1f2cd..32efa442 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java @@ -38,7 +38,6 @@ import com.amazonaws.services.kinesis.AmazonKinesisClient; import com.amazonaws.services.kinesis.clientlibrary.interfaces.ICheckpoint; import com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessor; import com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessorFactory; -import com.amazonaws.services.kinesis.clientlibrary.lib.worker.Worker.Builder; import com.amazonaws.services.kinesis.clientlibrary.proxies.KinesisProxyFactory; import com.amazonaws.services.kinesis.clientlibrary.types.ShutdownReason; import com.amazonaws.services.kinesis.leases.exceptions.LeasingException; @@ -47,6 +46,7 @@ import com.amazonaws.services.kinesis.metrics.impl.CWMetricsFactory; import com.amazonaws.services.kinesis.metrics.impl.NullMetricsFactory; import com.amazonaws.services.kinesis.metrics.interfaces.IMetricsFactory; import com.amazonaws.services.kinesis.metrics.interfaces.MetricsLevel; +import com.google.common.annotations.VisibleForTesting; /** * Worker is the high level class that Kinesis applications use to start @@ -342,46 +342,49 @@ public class Worker implements Runnable { } while (!shouldShutdown()) { - try { - boolean foundCompletedShard = false; - Set assignedShards = new HashSet(); - for (ShardInfo shardInfo : getShardInfoForAssignments()) { - ShardConsumer shardConsumer = createOrGetShardConsumer(shardInfo, recordProcessorFactory); - if (shardConsumer.isShutdown() - && shardConsumer.getShutdownReason().equals(ShutdownReason.TERMINATE)) { - foundCompletedShard = true; - } else { - shardConsumer.consumeShard(); - } - assignedShards.add(shardInfo); - } - - if (foundCompletedShard) { - controlServer.syncShardAndLeaseInfo(null); - } - - // clean up shard consumers for unassigned shards - cleanupShardConsumers(assignedShards); - - wlog.info("Sleeping ..."); - Thread.sleep(idleTimeInMilliseconds); - } catch (Exception e) { - LOG.error(String.format("Worker.run caught exception, sleeping for %s milli seconds!", - String.valueOf(idleTimeInMilliseconds)), - e); - try { - Thread.sleep(idleTimeInMilliseconds); - } catch (InterruptedException ex) { - LOG.info("Worker: sleep interrupted after catching exception ", ex); - } - } - wlog.resetInfoLogging(); + runProcessLoop(); } finalShutdown(); LOG.info("Worker loop is complete. Exiting from worker."); } + @VisibleForTesting + void runProcessLoop() { + try { + boolean foundCompletedShard = false; + Set assignedShards = new HashSet<>(); + for (ShardInfo shardInfo : getShardInfoForAssignments()) { + ShardConsumer shardConsumer = createOrGetShardConsumer(shardInfo, recordProcessorFactory); + if (shardConsumer.isShutdown() && shardConsumer.getShutdownReason().equals(ShutdownReason.TERMINATE)) { + foundCompletedShard = true; + } else { + shardConsumer.consumeShard(); + } + assignedShards.add(shardInfo); + } + + if (foundCompletedShard) { + controlServer.syncShardAndLeaseInfo(null); + } + + // clean up shard consumers for unassigned shards + cleanupShardConsumers(assignedShards); + + wlog.info("Sleeping ..."); + Thread.sleep(idleTimeInMilliseconds); + } catch (Exception e) { + LOG.error(String.format("Worker.run caught exception, sleeping for %s milli seconds!", + String.valueOf(idleTimeInMilliseconds)), e); + try { + Thread.sleep(idleTimeInMilliseconds); + } catch (InterruptedException ex) { + LOG.info("Worker: sleep interrupted after catching exception ", ex); + } + } + wlog.resetInfoLogging(); + } + private void initialize() { boolean isDone = false; Exception lastException = null; @@ -552,25 +555,22 @@ public class Worker implements Runnable { // completely processed (shutdown reason terminate). if ((consumer == null) || (consumer.isShutdown() && consumer.getShutdownReason().equals(ShutdownReason.ZOMBIE))) { - IRecordProcessor recordProcessor = factory.createProcessor(); - - consumer = - new ShardConsumer(shardInfo, - streamConfig, - checkpointTracker, - recordProcessor, - leaseCoordinator.getLeaseManager(), - parentShardPollIntervalMillis, - cleanupLeasesUponShardCompletion, - executorService, - metricsFactory, - taskBackoffTimeMillis); + consumer = buildConsumer(shardInfo, factory); shardInfoShardConsumerMap.put(shardInfo, consumer); wlog.infoForce("Created new shardConsumer for : " + shardInfo); } return consumer; } + protected ShardConsumer buildConsumer(ShardInfo shardInfo, IRecordProcessorFactory factory) { + IRecordProcessor recordProcessor = factory.createProcessor(); + + return new ShardConsumer(shardInfo, streamConfig, checkpointTracker, recordProcessor, + leaseCoordinator.getLeaseManager(), parentShardPollIntervalMillis, cleanupLeasesUponShardCompletion, + executorService, metricsFactory, taskBackoffTimeMillis); + + } + /** * Logger for suppressing too much INFO logging. To avoid too much logging * information Worker will output logging at INFO level for a single pass diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ParentsFirstShardPrioritizationUnitTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ParentsFirstShardPrioritizationUnitTest.java index 35b56b32..7ba0753d 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ParentsFirstShardPrioritizationUnitTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ParentsFirstShardPrioritizationUnitTest.java @@ -11,6 +11,8 @@ import java.util.Random; import org.junit.Test; +import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber; + public class ParentsFirstShardPrioritizationUnitTest { @Test(expected = IllegalArgumentException.class) @@ -144,17 +146,54 @@ public class ParentsFirstShardPrioritizationUnitTest { return "shardId-" + shardNumber; } + /** + * Builder class for ShardInfo. + */ + static class ShardInfoBuilder { + private String shardId; + private String concurrencyToken; + private List parentShardIds = Collections.emptyList(); + private ExtendedSequenceNumber checkpoint = ExtendedSequenceNumber.LATEST; + + ShardInfoBuilder() { + } + + ShardInfoBuilder withShardId(String shardId) { + this.shardId = shardId; + return this; + } + + ShardInfoBuilder withConcurrencyToken(String concurrencyToken) { + this.concurrencyToken = concurrencyToken; + return this; + } + + ShardInfoBuilder withParentShards(List parentShardIds) { + this.parentShardIds = parentShardIds; + return this; + } + + ShardInfoBuilder withCheckpoint(ExtendedSequenceNumber checkpoint) { + this.checkpoint = checkpoint; + return this; + } + + ShardInfo build() { + return new ShardInfo(shardId, concurrencyToken, parentShardIds, checkpoint); + } + } + private static ShardInfo shardInfo(String shardId, List parentShardIds) { // copy into new list just in case ShardInfo will stop doing it List newParentShardIds = new ArrayList<>(parentShardIds); - return new ShardInfo.Builder() + return new ShardInfoBuilder() .withShardId(shardId) .withParentShards(newParentShardIds) .build(); } private static ShardInfo shardInfo(String shardId, String... parentShardIds) { - return new ShardInfo.Builder() + return new ShardInfoBuilder() .withShardId(shardId) .withParentShards(Arrays.asList(parentShardIds)) .build(); diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardInfoTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardInfoTest.java index a2434d69..511b5a1b 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardInfoTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardInfoTest.java @@ -14,6 +14,10 @@ */ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -93,11 +97,20 @@ public class ShardInfoTest { } @Test - public void testPacboyShardInfoEqualsForCheckpoint() { - ShardInfo diffShardInfo = new ShardInfo(SHARD_ID, CONCURRENCY_TOKEN, parentShardIds, ExtendedSequenceNumber.SHARD_END); - Assert.assertFalse("Equal should return false with different checkpoint", diffShardInfo.equals(testShardInfo)); - diffShardInfo = new ShardInfo(SHARD_ID, CONCURRENCY_TOKEN, parentShardIds, null); - Assert.assertFalse("Equal should return false with null checkpoint", diffShardInfo.equals(testShardInfo)); + public void testShardInfoCheckpointEqualsHashCode() { + ShardInfo baseInfo = new ShardInfo(SHARD_ID, CONCURRENCY_TOKEN, parentShardIds, + ExtendedSequenceNumber.TRIM_HORIZON); + ShardInfo differentCheckpoint = new ShardInfo(SHARD_ID, CONCURRENCY_TOKEN, parentShardIds, + new ExtendedSequenceNumber("1234")); + ShardInfo nullCheckpoint = new ShardInfo(SHARD_ID, CONCURRENCY_TOKEN, parentShardIds, null); + + assertThat("Checkpoint should not be included in equality.", baseInfo.equals(differentCheckpoint), is(true)); + assertThat("Checkpoint should not be included in equality.", baseInfo.equals(nullCheckpoint), is(true)); + + assertThat("Checkpoint should not be included in hash code.", baseInfo.hashCode(), + equalTo(differentCheckpoint.hashCode())); + assertThat("Checkpoint should not be included in hash code.", baseInfo.hashCode(), + equalTo(nullCheckpoint.hashCode())); } @Test diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/WorkerTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/WorkerTest.java index a68c229b..0747f83d 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/WorkerTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/WorkerTest.java @@ -16,10 +16,16 @@ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Matchers.same; import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -50,7 +56,10 @@ import org.junit.Assert; import org.junit.Rule; import org.junit.Test; import org.junit.rules.Timeout; +import org.junit.runner.RunWith; +import org.mockito.Mock; import org.mockito.invocation.InvocationOnMock; +import org.mockito.runners.MockitoJUnitRunner; import org.mockito.stubbing.Answer; import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; @@ -85,6 +94,7 @@ import com.amazonaws.services.kinesis.model.Shard; /** * Unit tests of Worker. */ +@RunWith(MockitoJUnitRunner.class) public class WorkerTest { private static final Log LOG = LogFactory.getLog(WorkerTest.class); @@ -107,6 +117,30 @@ public class WorkerTest { InitialPositionInStreamExtended.newInitialPosition(InitialPositionInStream.TRIM_HORIZON); private final ShardPrioritization shardPrioritization = new NoOpShardPrioritization(); + private static final String KINESIS_SHARD_ID_FORMAT = "kinesis-0-0-%d"; + private static final String CONCURRENCY_TOKEN_FORMAT = "testToken-%d"; + + @Mock + private KinesisClientLibLeaseCoordinator leaseCoordinator; + @Mock + private ILeaseManager leaseManager; + @Mock + private com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorFactory v1RecordProcessorFactory; + @Mock + private IKinesisProxy proxy; + @Mock + private WorkerThreadPoolExecutor executorService; + @Mock + private WorkerCWMetricsFactory cwMetricsFactory; + @Mock + private IKinesisProxy kinesisProxy; + @Mock + private IRecordProcessorFactory v2RecordProcessorFactory; + @Mock + private IRecordProcessor v2RecordProcessor; + @Mock + private ShardConsumer shardConsumer; + // CHECKSTYLE:IGNORE AnonInnerLengthCheck FOR NEXT 50 LINES private static final com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorFactory SAMPLE_RECORD_PROCESSOR_FACTORY = new com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorFactory() { @@ -145,6 +179,7 @@ public class WorkerTest { private static final IRecordProcessorFactory SAMPLE_RECORD_PROCESSOR_FACTORY_V2 = new V1ToV2RecordProcessorFactoryAdapter(SAMPLE_RECORD_PROCESSOR_FACTORY); + /** * Test method for {@link com.amazonaws.services.kinesis.clientlibrary.lib.worker.Worker#getApplicationName()}. */ @@ -153,9 +188,7 @@ public class WorkerTest { final String stageName = "testStageName"; final KinesisClientLibConfiguration clientConfig = new KinesisClientLibConfiguration(stageName, null, null, null); - Worker worker = - new Worker(mock(com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorFactory.class), - clientConfig); + Worker worker = new Worker(v1RecordProcessorFactory, clientConfig); Assert.assertEquals(stageName, worker.getApplicationName()); } @@ -177,9 +210,6 @@ public class WorkerTest { final String dummyKinesisShardId = "kinesis-0-0"; ExecutorService execService = null; - KinesisClientLibLeaseCoordinator leaseCoordinator = mock(KinesisClientLibLeaseCoordinator.class); - @SuppressWarnings("unchecked") - ILeaseManager leaseManager = mock(ILeaseManager.class); when(leaseCoordinator.getLeaseManager()).thenReturn(leaseManager); Worker worker = @@ -208,6 +238,63 @@ public class WorkerTest { Assert.assertNotSame(consumer3, consumer); } + @Test + public void testWorkerLoopWithCheckpoint() { + final String stageName = "testStageName"; + IRecordProcessorFactory streamletFactory = SAMPLE_RECORD_PROCESSOR_FACTORY_V2; + IKinesisProxy proxy = null; + ICheckpoint checkpoint = null; + int maxRecords = 1; + int idleTimeInMilliseconds = 1000; + StreamConfig streamConfig = new StreamConfig(proxy, maxRecords, idleTimeInMilliseconds, + callProcessRecordsForEmptyRecordList, skipCheckpointValidationValue, INITIAL_POSITION_LATEST); + + ExecutorService execService = null; + + when(leaseCoordinator.getLeaseManager()).thenReturn(leaseManager); + + List initialState = createShardInfoList(ExtendedSequenceNumber.TRIM_HORIZON); + List firstCheckpoint = createShardInfoList(new ExtendedSequenceNumber("1000")); + List secondCheckpoint = createShardInfoList(new ExtendedSequenceNumber("2000")); + + when(leaseCoordinator.getCurrentAssignments()).thenReturn(initialState).thenReturn(firstCheckpoint) + .thenReturn(secondCheckpoint); + + Worker worker = new Worker(stageName, streamletFactory, streamConfig, INITIAL_POSITION_LATEST, + parentShardPollIntervalMillis, shardSyncIntervalMillis, cleanupLeasesUponShardCompletion, checkpoint, + leaseCoordinator, execService, nullMetricsFactory, taskBackoffTimeMillis, failoverTimeMillis, + shardPrioritization); + + Worker workerSpy = spy(worker); + + doReturn(shardConsumer).when(workerSpy).buildConsumer(eq(initialState.get(0)), any(IRecordProcessorFactory.class)); + workerSpy.runProcessLoop(); + workerSpy.runProcessLoop(); + workerSpy.runProcessLoop(); + + verify(workerSpy).buildConsumer(same(initialState.get(0)), any(IRecordProcessorFactory.class)); + verify(workerSpy, never()).buildConsumer(same(firstCheckpoint.get(0)), any(IRecordProcessorFactory.class)); + verify(workerSpy, never()).buildConsumer(same(secondCheckpoint.get(0)), any(IRecordProcessorFactory.class)); + + } + + private List createShardInfoList(ExtendedSequenceNumber... sequenceNumbers) { + List result = new ArrayList<>(sequenceNumbers.length); + assertThat(sequenceNumbers.length, greaterThanOrEqualTo(1)); + for (int i = 0; i < sequenceNumbers.length; ++i) { + result.add(new ShardInfo(adjustedShardId(i), adjustedConcurrencyToken(i), null, sequenceNumbers[i])); + } + return result; + } + + private String adjustedShardId(int index) { + return String.format(KINESIS_SHARD_ID_FORMAT, index); + } + + private String adjustedConcurrencyToken(int index) { + return String.format(CONCURRENCY_TOKEN_FORMAT, index); + } + @Test public final void testCleanupShardConsumers() { final String stageName = "testStageName"; @@ -226,10 +313,6 @@ public class WorkerTest { final String dummyKinesisShardId = "kinesis-0-0"; final String anotherDummyKinesisShardId = "kinesis-0-1"; ExecutorService execService = null; - - KinesisClientLibLeaseCoordinator leaseCoordinator = mock(KinesisClientLibLeaseCoordinator.class); - @SuppressWarnings("unchecked") - ILeaseManager leaseManager = mock(ILeaseManager.class); when(leaseCoordinator.getLeaseManager()).thenReturn(leaseManager); Worker worker = @@ -272,7 +355,6 @@ public class WorkerTest { public final void testInitializationFailureWithRetries() { String stageName = "testInitializationWorker"; IRecordProcessorFactory recordProcessorFactory = new TestStreamletFactory(null, null); - IKinesisProxy proxy = mock(IKinesisProxy.class); int count = 0; when(proxy.getShardList()).thenThrow(new RuntimeException(Integer.toString(count++))); int maxRecords = 2; @@ -282,9 +364,6 @@ public class WorkerTest { maxRecords, idleTimeInMilliseconds, callProcessRecordsForEmptyRecordList, skipCheckpointValidationValue, INITIAL_POSITION_LATEST); - KinesisClientLibLeaseCoordinator leaseCoordinator = mock(KinesisClientLibLeaseCoordinator.class); - @SuppressWarnings("unchecked") - ILeaseManager leaseManager = mock(ILeaseManager.class); when(leaseCoordinator.getLeaseManager()).thenReturn(leaseManager); ExecutorService execService = Executors.newSingleThreadExecutor(); long shardPollInterval = 0L; @@ -374,8 +453,7 @@ public class WorkerTest { @Test public final void testWorkerShutsDownOwnedResources() throws Exception { - final WorkerThreadPoolExecutor executorService = mock(WorkerThreadPoolExecutor.class); - final WorkerCWMetricsFactory cwMetricsFactory = mock(WorkerCWMetricsFactory.class); + final long failoverTimeMillis = 20L; // Make sure that worker thread is run before invoking shutdown. @@ -393,8 +471,7 @@ public class WorkerTest { callProcessRecordsForEmptyRecordList, failoverTimeMillis, 10, - mock(IKinesisProxy.class), - mock(IRecordProcessorFactory.class), + kinesisProxy, v2RecordProcessorFactory, executorService, cwMetricsFactory); @@ -411,10 +488,10 @@ public class WorkerTest { @Test public final void testWorkerDoesNotShutdownClientResources() throws Exception { - final ExecutorService executorService = mock(ThreadPoolExecutor.class); - final CWMetricsFactory cwMetricsFactory = mock(CWMetricsFactory.class); final long failoverTimeMillis = 20L; + final ExecutorService executorService = mock(ThreadPoolExecutor.class); + final CWMetricsFactory cwMetricsFactory = mock(CWMetricsFactory.class); // Make sure that worker thread is run before invoking shutdown. final CountDownLatch workerStarted = new CountDownLatch(1); doAnswer(new Answer() { @@ -430,8 +507,7 @@ public class WorkerTest { callProcessRecordsForEmptyRecordList, failoverTimeMillis, 10, - mock(IKinesisProxy.class), - mock(IRecordProcessorFactory.class), + kinesisProxy, v2RecordProcessorFactory, executorService, cwMetricsFactory); @@ -468,9 +544,8 @@ public class WorkerTest { // Make test case as efficient as possible. final CountDownLatch processRecordsLatch = new CountDownLatch(1); - IRecordProcessorFactory recordProcessorFactory = mock(IRecordProcessorFactory.class); - IRecordProcessor recordProcessor = mock(IRecordProcessor.class); - when(recordProcessorFactory.createProcessor()).thenReturn(recordProcessor); + + when(v2RecordProcessorFactory.createProcessor()).thenReturn(v2RecordProcessor); doAnswer(new Answer () { @Override @@ -479,7 +554,7 @@ public class WorkerTest { processRecordsLatch.countDown(); return null; } - }).when(recordProcessor).processRecords(any(ProcessRecordsInput.class)); + }).when(v2RecordProcessor).processRecords(any(ProcessRecordsInput.class)); WorkerThread workerThread = runWorker(shardList, initialLeases, @@ -487,7 +562,7 @@ public class WorkerTest { failoverTimeMillis, numberOfRecordsPerShard, fileBasedProxy, - recordProcessorFactory, + v2RecordProcessorFactory, executorService, nullMetricsFactory); @@ -495,16 +570,16 @@ public class WorkerTest { processRecordsLatch.await(); // Make sure record processor is initialized and processing records. - verify(recordProcessorFactory, times(1)).createProcessor(); - verify(recordProcessor, times(1)).initialize(any(InitializationInput.class)); - verify(recordProcessor, atLeast(1)).processRecords(any(ProcessRecordsInput.class)); - verify(recordProcessor, times(0)).shutdown(any(ShutdownInput.class)); + verify(v2RecordProcessorFactory, times(1)).createProcessor(); + verify(v2RecordProcessor, times(1)).initialize(any(InitializationInput.class)); + verify(v2RecordProcessor, atLeast(1)).processRecords(any(ProcessRecordsInput.class)); + verify(v2RecordProcessor, times(0)).shutdown(any(ShutdownInput.class)); workerThread.getWorker().shutdown(); workerThread.join(); Assert.assertTrue(workerThread.getState() == State.TERMINATED); - verify(recordProcessor, times(1)).shutdown(any(ShutdownInput.class)); + verify(v2RecordProcessor, times(1)).shutdown(any(ShutdownInput.class)); } /** @@ -538,9 +613,7 @@ public class WorkerTest { // Make test case as efficient as possible. final CountDownLatch processRecordsLatch = new CountDownLatch(1); final AtomicBoolean recordProcessorInterrupted = new AtomicBoolean(false); - IRecordProcessorFactory recordProcessorFactory = mock(IRecordProcessorFactory.class); - IRecordProcessor recordProcessor = mock(IRecordProcessor.class); - when(recordProcessorFactory.createProcessor()).thenReturn(recordProcessor); + when(v2RecordProcessorFactory.createProcessor()).thenReturn(v2RecordProcessor); final Semaphore actionBlocker = new Semaphore(1); final Semaphore shutdownBlocker = new Semaphore(1); @@ -572,7 +645,7 @@ public class WorkerTest { return null; } - }).when(recordProcessor).processRecords(any(ProcessRecordsInput.class)); + }).when(v2RecordProcessor).processRecords(any(ProcessRecordsInput.class)); WorkerThread workerThread = runWorker(shardList, initialLeases, @@ -580,7 +653,7 @@ public class WorkerTest { failoverTimeMillis, numberOfRecordsPerShard, fileBasedProxy, - recordProcessorFactory, + v2RecordProcessorFactory, executorService, nullMetricsFactory); @@ -588,17 +661,17 @@ public class WorkerTest { processRecordsLatch.await(); // Make sure record processor is initialized and processing records. - verify(recordProcessorFactory, times(1)).createProcessor(); - verify(recordProcessor, times(1)).initialize(any(InitializationInput.class)); - verify(recordProcessor, atLeast(1)).processRecords(any(ProcessRecordsInput.class)); - verify(recordProcessor, times(0)).shutdown(any(ShutdownInput.class)); + verify(v2RecordProcessorFactory, times(1)).createProcessor(); + verify(v2RecordProcessor, times(1)).initialize(any(InitializationInput.class)); + verify(v2RecordProcessor, atLeast(1)).processRecords(any(ProcessRecordsInput.class)); + verify(v2RecordProcessor, times(0)).shutdown(any(ShutdownInput.class)); workerThread.getWorker().shutdown(); workerThread.join(); Assert.assertTrue(workerThread.getState() == State.TERMINATED); // Shutdown should not be called in this case because record processor is blocked. - verify(recordProcessor, times(0)).shutdown(any(ShutdownInput.class)); + verify(v2RecordProcessor, times(0)).shutdown(any(ShutdownInput.class)); // // Release the worker thread From 4cbb6d851e203c36604e6cdde07ca63409d8e267 Mon Sep 17 00:00:00 2001 From: "Pfifer, Justin" Date: Thu, 18 Aug 2016 08:41:24 -0700 Subject: [PATCH 7/9] Prevent Spurious Info Message on getIterator Clean up check for ShardIteratorType to prevent emitting a spurious message for every call to GetIterator Add missing KinesisProxyTest, and add 3 new tests for getIterator. --- .../clientlibrary/proxies/KinesisProxy.java | 15 +- .../proxies/KinesisProxyTest.java | 167 ++++++++++++++++++ 2 files changed, 180 insertions(+), 2 deletions(-) create mode 100644 src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxyTest.java diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxy.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxy.java index ad929c21..de330dc9 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxy.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxy.java @@ -17,6 +17,7 @@ package com.amazonaws.services.kinesis.clientlibrary.proxies; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Date; +import java.util.EnumSet; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -51,6 +52,9 @@ public class KinesisProxy implements IKinesisProxyExtended { private static final Log LOG = LogFactory.getLog(KinesisProxy.class); + private static final EnumSet EXPECTED_ITERATOR_TYPES = EnumSet + .of(ShardIteratorType.AT_SEQUENCE_NUMBER, ShardIteratorType.AFTER_SEQUENCE_NUMBER); + private static String defaultServiceName = "kinesis"; private static String defaultRegionId = "us-east-1";; @@ -265,8 +269,15 @@ public class KinesisProxy implements IKinesisProxyExtended { */ @Override public String getIterator(String shardId, String iteratorType, String sequenceNumber) { - if (!iteratorType.equals(ShardIteratorType.AT_SEQUENCE_NUMBER.toString()) || !iteratorType.equals( - ShardIteratorType.AFTER_SEQUENCE_NUMBER.toString())) { + ShardIteratorType shardIteratorType; + try { + shardIteratorType = ShardIteratorType.fromValue(iteratorType); + } catch (IllegalArgumentException iae) { + LOG.error("Caught illegal argument exception while parsing iteratorType: " + iteratorType, iae); + shardIteratorType = null; + } + + if (!EXPECTED_ITERATOR_TYPES.contains(shardIteratorType)) { LOG.info("This method should only be used for AT_SEQUENCE_NUMBER and AFTER_SEQUENCE_NUMBER " + "ShardIteratorTypes. For methods to use with other ShardIteratorTypes, see IKinesisProxy.java"); } diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxyTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxyTest.java new file mode 100644 index 00000000..2c1107b2 --- /dev/null +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxyTest.java @@ -0,0 +1,167 @@ +package com.amazonaws.services.kinesis.clientlibrary.proxies; + +import static org.hamcrest.Matchers.both; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasProperty; +import static org.hamcrest.Matchers.isA; +import static org.hamcrest.Matchers.nullValue; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.argThat; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import com.amazonaws.AmazonServiceException; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentMatcher; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; + +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.services.kinesis.AmazonKinesisClient; +import com.amazonaws.services.kinesis.model.DescribeStreamRequest; +import com.amazonaws.services.kinesis.model.DescribeStreamResult; +import com.amazonaws.services.kinesis.model.GetShardIteratorRequest; +import com.amazonaws.services.kinesis.model.GetShardIteratorResult; +import com.amazonaws.services.kinesis.model.LimitExceededException; +import com.amazonaws.services.kinesis.model.Shard; +import com.amazonaws.services.kinesis.model.ShardIteratorType; +import com.amazonaws.services.kinesis.model.StreamDescription; +import com.amazonaws.services.kinesis.model.StreamStatus; + +import junit.framework.Assert; + +@RunWith(MockitoJUnitRunner.class) +public class KinesisProxyTest { + private static final String TEST_STRING = "TestString"; + private static final long BACKOFF_TIME = 10L; + private static final int RETRY_TIMES = 50; + + @Mock + private AmazonKinesisClient mockClient; + @Mock + private AWSCredentialsProvider mockCredentialsProvider; + @Mock + private GetShardIteratorResult shardIteratorResult; + private KinesisProxy proxy; + + // Test shards for verifying. + private Set shardIdSet; + private List shards; + + @Before + public void setUpTest() { + // Set up kinesis proxy + proxy = new KinesisProxy(TEST_STRING, mockCredentialsProvider, mockClient, BACKOFF_TIME, RETRY_TIMES); + when(mockCredentialsProvider.getCredentials()).thenReturn(null); + // Set up test shards + shardIdSet = new HashSet<>(); + shards = new ArrayList<>(); + String[] shardIds = new String[] { "shard-1", "shard-2", "shard-3", "shard-4" }; + for (String shardId : shardIds) { + Shard shard = new Shard(); + shard.setShardId(shardId); + shards.add(shard); + shardIdSet.add(shardId); + } + } + + @Test + public void testGetShardListWithMoreDataAvailable() { + // Set up mock : + // First call describeStream returning response with first two shards in the list; + // Second call describeStream returning response with rest shards. + DescribeStreamResult responseWithMoreData = createGetStreamInfoResponse(shards.subList(0, 2), true); + DescribeStreamResult responseFinal = createGetStreamInfoResponse(shards.subList(2, shards.size()), false); + doReturn(responseWithMoreData).when(mockClient).describeStream(argThat(new IsRequestWithStartShardId(null))); + doReturn(responseFinal).when(mockClient) + .describeStream(argThat(new IsRequestWithStartShardId(shards.get(1).getShardId()))); + + Set resultShardIdSets = proxy.getAllShardIds(); + Assert.assertTrue("Result set should equal to Test set", shardIdSet.equals(resultShardIdSets)); + } + + @Test + public void testGetShardListWithLimitExceededException() { + // Set up mock : + // First call describeStream throwing LimitExceededException; + // Second call describeStream returning shards list. + DescribeStreamResult response = createGetStreamInfoResponse(shards, false); + doThrow(new LimitExceededException("Test Exception")).doReturn(response).when(mockClient) + .describeStream(argThat(new IsRequestWithStartShardId(null))); + + Set resultShardIdSet = proxy.getAllShardIds(); + Assert.assertTrue("Result set should equal to Test set", shardIdSet.equals(resultShardIdSet)); + } + + @Test + public void testValidShardIteratorType() { + when(mockClient.getShardIterator(any(GetShardIteratorRequest.class))).thenReturn(shardIteratorResult); + String expectedShardIteratorType = ShardIteratorType.AFTER_SEQUENCE_NUMBER.toString(); + proxy.getIterator("Shard-001", expectedShardIteratorType, "1234"); + + verify(mockClient).getShardIterator(argThat(both(isA(GetShardIteratorRequest.class)) + .and(hasProperty("shardIteratorType", equalTo(expectedShardIteratorType))))); + } + + @Test + public void testInvalidShardIteratorIsntChanged() { + when(mockClient.getShardIterator(any(GetShardIteratorRequest.class))).thenReturn(shardIteratorResult); + String expectedShardIteratorType = ShardIteratorType.AT_TIMESTAMP.toString(); + proxy.getIterator("Shard-001", expectedShardIteratorType, "1234"); + + verify(mockClient).getShardIterator(argThat(both(isA(GetShardIteratorRequest.class)) + .and(hasProperty("shardIteratorType", equalTo(expectedShardIteratorType))))); + } + + @Test(expected = AmazonServiceException.class) + public void testNullShardIteratorType() { + when(mockClient.getShardIterator(any(GetShardIteratorRequest.class))).thenThrow(new AmazonServiceException("expected null")); + String expectedShardIteratorType = null; + proxy.getIterator("Shard-001", expectedShardIteratorType, "1234"); + + verify(mockClient).getShardIterator(argThat(both(isA(GetShardIteratorRequest.class)) + .and(hasProperty("shardIteratorType", nullValue(String.class))))); + } + + private DescribeStreamResult createGetStreamInfoResponse(List shards1, boolean isHasMoreShards) { + // Create stream description + StreamDescription description = new StreamDescription(); + description.setHasMoreShards(isHasMoreShards); + description.setShards(shards1); + description.setStreamStatus(StreamStatus.ACTIVE); + + // Create Describe Stream Result + DescribeStreamResult response = new DescribeStreamResult(); + response.setStreamDescription(description); + return response; + } + + // Matcher for testing describe stream request with specific start shard ID. + private static class IsRequestWithStartShardId extends ArgumentMatcher { + private final String shardId; + + public IsRequestWithStartShardId(String shardId) { + this.shardId = shardId; + } + + @Override + public boolean matches(Object request) { + String startShardId = ((DescribeStreamRequest) request).getExclusiveStartShardId(); + // If startShardId equals to null, shardId should also be null. + if (startShardId == null) { + return shardId == null; + } + return startShardId.equals(shardId); + } + } + +} From 8a32e89fe93276b4aa6848734cb14eea564351a1 Mon Sep 17 00:00:00 2001 From: "Pfifer, Justin" Date: Mon, 22 Aug 2016 12:55:19 -0700 Subject: [PATCH 8/9] Version 1.7.0 of the Amazon Kinesis Client Library * Add support for time based iterators ([See GetShardIterator Documentation](http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetShardIterator.html)) * [PR #94](https://github.com/awslabs/amazon-kinesis-client/pull/94) The `KinesisClientLibConfiguration` now supports providing an initial time stamp position. * This position is only used if there is no current checkpoint for the shard. * This setting cannot be used with DynamoDB Streams Resolves [Issue #88](https://github.com/awslabs/amazon-kinesis-client/issues/88) * Allow Prioritization of Parent Shards for Task Assignment * [PR #95](https://github.com/awslabs/amazon-kinesis-client/pull/95) The `KinesisClientLibconfiguration` now supports providing a `ShardPrioritization` strategy. This strategy controls how the `Worker` determines which `ShardConsumer` to call next. This can improve processing for streams that split often, such as DynamoDB Streams. * Remove direct dependency on `aws-java-sdk-core`, to allow independent versioning. * [PR #92](https://github.com/awslabs/amazon-kinesis-client/pull/92) **You may need to add a direct dependency on aws-java-sdk-core if other dependencies include an older version.** --- README.md | 15 +++++++++++++-- pom.xml | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index cdc7dd7d..e3a1e63a 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,19 @@ For producer-side developers using the **[Kinesis Producer Library (KPL)][kinesi To make it easier for developers to write record processors in other languages, we have implemented a Java based daemon, called MultiLangDaemon that does all the heavy lifting. Our approach has the daemon spawn a sub-process, which in turn runs the record processor, which can be written in any language. The MultiLangDaemon process and the record processor sub-process communicate with each other over [STDIN and STDOUT using a defined protocol][multi-lang-protocol]. There will be a one to one correspondence amongst record processors, child processes, and shards. For Python developers specifically, we have abstracted these implementation details away and [expose an interface][kclpy] that enables you to focus on writing record processing logic in Python. This approach enables KCL to be language agnostic, while providing identical features and similar parallel processing model across all languages. ## Release Notes -### Release 1.7.0 (???) -* Place holder for release 1.7.0 +### Release 1.7.0 (August 22, 2016) +* Add support for time based iterators ([See GetShardIterator Documentation](http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetShardIterator.html)) + * [PR #94](https://github.com/awslabs/amazon-kinesis-client/pull/94) + The `KinesisClientLibConfiguration` now supports providing an initial time stamp position. + * This position is only used if there is no current checkpoint for the shard. + * This setting cannot be used with DynamoDB Streams + Resolves [Issue #88](https://github.com/awslabs/amazon-kinesis-client/issues/88) +* Allow Prioritization of Parent Shards for Task Assignment + * [PR #95](https://github.com/awslabs/amazon-kinesis-client/pull/95) + The `KinesisClientLibconfiguration` now supports providing a `ShardPrioritization` strategy. This strategy controls how the `Worker` determines which `ShardConsumer` to call next. This can improve processing for streams that split often, such as DynamoDB Streams. +* Remove direct dependency on `aws-java-sdk-core`, to allow independent versioning. + * [PR #92](https://github.com/awslabs/amazon-kinesis-client/pull/92) + **You may need to add a direct dependency on aws-java-sdk-core if other dependencies include an older version.** ### Release 1.6.5 (July 25, 2016) * Change LeaseManager to call DescribeTable before attempting to create the lease table. diff --git a/pom.xml b/pom.xml index c9291397..626dac7f 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ amazon-kinesis-client jar Amazon Kinesis Client Library for Java - 1.7.0-SNAPSHOT + 1.7.0 The Amazon Kinesis Client Library for Java enables Java developers to easily consume and process data from Amazon Kinesis. From 51663f96c712da74daeeee2131953c8d8a515cc4 Mon Sep 17 00:00:00 2001 From: "Pfifer, Justin" Date: Thu, 1 Sep 2016 07:36:20 -0700 Subject: [PATCH 9/9] Move Version to Next Development Version Adjusted the version to 1.7.1-SNAPSHOT for current development. --- META-INF/MANIFEST.MF | 2 +- pom.xml | 2 +- .../clientlibrary/lib/worker/KinesisClientLibConfiguration.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/META-INF/MANIFEST.MF b/META-INF/MANIFEST.MF index f6675044..25c30916 100644 --- a/META-INF/MANIFEST.MF +++ b/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: Amazon Kinesis Client Library for Java Bundle-SymbolicName: com.amazonaws.kinesisclientlibrary;singleton:=true -Bundle-Version: 1.7.0 +Bundle-Version: 1.7.1-SNAPSHOT Bundle-Vendor: Amazon Technologies, Inc Bundle-RequiredExecutionEnvironment: JavaSE-1.7 Require-Bundle: org.apache.commons.codec;bundle-version="1.6", diff --git a/pom.xml b/pom.xml index 626dac7f..a68997cc 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ amazon-kinesis-client jar Amazon Kinesis Client Library for Java - 1.7.0 + 1.7.1-SNAPSHOT The Amazon Kinesis Client Library for Java enables Java developers to easily consume and process data from Amazon Kinesis. diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java index 590382b4..0f930053 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java @@ -120,7 +120,7 @@ public class KinesisClientLibConfiguration { /** * User agent set when Amazon Kinesis Client Library makes AWS requests. */ - public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.7.0"; + public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.7.1-SNAPSHOT"; /** * KCL will validate client provided sequence numbers with a call to Amazon Kinesis before checkpointing for calls