From 43bbeb5fe69ba8073fa1fc90148d98e253d26e5e Mon Sep 17 00:00:00 2001 From: glarwood Date: Wed, 12 Dec 2018 13:58:35 -0800 Subject: [PATCH] revert to release 1.9.3 --- .github/PULL_REQUEST_TEMPLATE.md | 6 + .travis.yml | 6 +- CHANGELOG.md | 329 +++++ CODE_OF_CONDUCT.md | 4 + CONTRIBUTING.md | 61 + META-INF/MANIFEST.MF | 6 +- README.md | 149 +-- build.properties | 10 - pom.xml | 86 +- .../config/DatePropertyValueDecorder.java | 53 + .../config/KinesisClientLibConfigurator.java | 1 + .../clientlibrary/interfaces/ICheckpoint.java | 27 + .../interfaces/IPreparedCheckpointer.java | 41 + .../IRecordProcessorCheckpointer.java | 107 ++ .../interfaces/v2/IRecordProcessor.java | 12 +- .../v2/IShutdownNotificationAware.java | 14 + .../lib/checkpoint/Checkpoint.java | 27 + ...ynchronousGetRecordsRetrievalStrategy.java | 154 +++ .../lib/worker/BlockingGetRecordsCache.java | 65 + .../lib/worker/ConsumerStates.java | 55 +- .../lib/worker/DataFetcherResult.java | 37 + .../lib/worker/DataFetchingStrategy.java | 8 + .../DoesNothingPreparedCheckpointer.java | 66 + .../lib/worker/GetRecordsCache.java | 43 + .../worker/GetRecordsRetrievalStrategy.java | 54 + .../lib/worker/GetRecordsRetriever.java | 12 + .../lib/worker/GracefulShutdownContext.java | 33 + .../worker/GracefulShutdownCoordinator.java | 163 +++ .../lib/worker/InitializeTask.java | 41 +- .../worker/KinesisClientLibConfiguration.java | 470 ++++++- .../KinesisClientLibLeaseCoordinator.java | 75 +- .../lib/worker/KinesisDataFetcher.java | 108 +- .../lib/worker/NoOpShardPrioritization.java | 14 + .../worker/NoOpWorkerStateChangeListener.java | 16 + .../ParentsFirstShardPrioritization.java | 14 + .../lib/worker/PrefetchGetRecordsCache.java | 268 ++++ .../lib/worker/PreparedCheckpointer.java | 65 + .../clientlibrary/lib/worker/ProcessTask.java | 83 +- .../worker/RecordProcessorCheckpointer.java | 184 ++- .../lib/worker/RecordsFetcherFactory.java | 76 ++ .../lib/worker/SequenceNumberValidator.java | 6 +- .../lib/worker/ShardConsumer.java | 204 ++- .../ShardConsumerShutdownNotification.java | 14 + .../clientlibrary/lib/worker/ShardInfo.java | 22 +- .../lib/worker/ShardPrioritization.java | 14 + .../lib/worker/ShardSyncTask.java | 6 +- .../lib/worker/ShardSyncTaskManager.java | 5 + .../clientlibrary/lib/worker/ShardSyncer.java | 114 +- .../lib/worker/ShutdownFuture.java | 155 --- .../lib/worker/ShutdownNotification.java | 14 + .../lib/worker/ShutdownNotificationTask.java | 14 + .../lib/worker/ShutdownReason.java | 18 +- .../lib/worker/ShutdownTask.java | 47 +- .../worker/SimpleRecordsFetcherFactory.java | 74 ++ ...ynchronousGetRecordsRetrievalStrategy.java | 50 + .../lib/worker/ThrottlingReporter.java | 14 + .../clientlibrary/lib/worker/Worker.java | 1172 ++++++++++------- .../lib/worker/WorkerStateChangeListener.java | 16 + .../clientlibrary/proxies/IKinesisProxy.java | 7 + .../proxies/IKinesisProxyFactory.java | 4 + .../clientlibrary/proxies/KinesisProxy.java | 429 ++++-- .../proxies/KinesisProxyFactory.java | 53 +- .../types/InitializationInput.java | 41 +- .../kinesis/clientlibrary/types/Messages.java | 15 + .../types/ProcessRecordsInput.java | 45 +- .../clientlibrary/types/ShutdownInput.java | 18 +- .../clientlibrary/types/UserRecord.java | 2 +- .../utils/NamedThreadFactory.java | 14 + .../leases/impl/KinesisClientLease.java | 48 +- .../impl/KinesisClientLeaseSerializer.java | 42 +- .../kinesis/leases/impl/LeaseCoordinator.java | 26 +- .../kinesis/leases/impl/LeaseManager.java | 34 +- .../kinesis/metrics/impl/MetricsHelper.java | 11 +- .../kinesis/multilang/MessageWriter.java | 14 +- .../kinesis/multilang/MultiLangDaemon.java | 38 +- .../multilang/MultiLangDaemonConfig.java | 124 +- .../kinesis/multilang/MultiLangProtocol.java | 137 +- .../multilang/MultiLangRecordProcessor.java | 53 +- .../MultiLangRecordProcessorFactory.java | 33 +- .../multilang/messages/CheckpointMessage.java | 18 +- .../multilang/messages/InitializeMessage.java | 18 +- .../messages/JsonFriendlyRecord.java | 18 +- .../kinesis/multilang/messages/Message.java | 25 +- .../messages/ProcessRecordsMessage.java | 18 +- .../messages/ShutdownRequestedMessage.java | 31 + .../AmazonDynamoDBStreamsAdapterClient.java | 26 + ...azonDynamoDBStreamsAdapterClientChild.java | 23 + .../config/DatePropertyValueDecoderTest.java | 53 + .../KinesisClientLibConfiguratorTest.java | 123 +- .../checkpoint/CheckpointImplTestBase.java | 62 + .../checkpoint/InMemoryCheckpointImpl.java | 18 + ...cordsRetrievalStrategyIntegrationTest.java | 193 +++ ...ronousGetRecordsRetrievalStrategyTest.java | 184 +++ .../worker/BlockingGetRecordsCacheTest.java | 83 ++ .../lib/worker/ConsumerStatesTest.java | 86 +- .../GracefulShutdownCoordinatorTest.java | 322 +++++ .../KinesisClientLibConfigurationTest.java | 101 +- .../lib/worker/KinesisDataFetcherTest.java | 193 ++- ...rentsFirstShardPrioritizationUnitTest.java | 14 + ...refetchGetRecordsCacheIntegrationTest.java | 222 ++++ .../worker/PrefetchGetRecordsCacheTest.java | 236 ++++ .../lib/worker/PreparedCheckpointerTest.java | 49 + .../lib/worker/ProcessTaskTest.java | 50 +- .../RecordProcessorCheckpointerTest.java | 564 +++++++- .../lib/worker/RecordsFetcherFactoryTest.java | 43 + .../worker/SequenceNumberValidatorTest.java | 2 +- .../lib/worker/ShardConsumerTest.java | 472 ++++++- .../worker/ShardSyncTaskIntegrationTest.java | 3 +- .../lib/worker/ShardSyncerTest.java | 77 +- .../lib/worker/ShutdownFutureTest.java | 236 ---- .../lib/worker/ShutdownTaskTest.java | 41 +- .../lib/worker/ThrottlingReporterTest.java | 14 + .../clientlibrary/lib/worker/WorkerTest.java | 748 +++++++++-- .../proxies/KinesisLocalFileProxyFactory.java | 2 - .../proxies/KinesisProxyTest.java | 528 +++++++- .../types/ShutdownReasonTest.java | 14 + .../impl/KinesisClientLeaseBuilder.java | 22 +- .../leases/impl/LeaseIntegrationTest.java | 19 +- .../impl/LeaseManagerIntegrationTest.java | 21 +- .../impl/LeaseRenewerIntegrationTest.java | 49 +- .../leases/impl/TestHarnessBuilder.java | 33 +- .../services/kinesis/multilang/Matchers.java | 14 + .../kinesis/multilang/MessageWriterTest.java | 28 +- .../multilang/MultiLangDaemonConfigTest.java | 121 +- .../multilang/MultiLangProtocolTest.java | 153 ++- .../StreamingRecordProcessorFactoryTest.java | 28 +- .../StreamingRecordProcessorTest.java | 114 +- .../multilang/messages/MessageTest.java | 20 +- src/test/{java => resources}/log4j.properties | 0 129 files changed, 9557 insertions(+), 2165 deletions(-) create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 CHANGELOG.md create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md delete mode 100644 build.properties create mode 100644 src/main/java/com/amazonaws/services/kinesis/clientlibrary/config/DatePropertyValueDecorder.java create mode 100644 src/main/java/com/amazonaws/services/kinesis/clientlibrary/interfaces/IPreparedCheckpointer.java create mode 100644 src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/Checkpoint.java create mode 100644 src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/AsynchronousGetRecordsRetrievalStrategy.java create mode 100644 src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/BlockingGetRecordsCache.java create mode 100644 src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/DataFetcherResult.java create mode 100644 src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/DataFetchingStrategy.java create mode 100644 src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/DoesNothingPreparedCheckpointer.java create mode 100644 src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/GetRecordsCache.java create mode 100644 src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/GetRecordsRetrievalStrategy.java create mode 100644 src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/GetRecordsRetriever.java create mode 100644 src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/GracefulShutdownContext.java create mode 100644 src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/GracefulShutdownCoordinator.java create mode 100644 src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/NoOpWorkerStateChangeListener.java create mode 100644 src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PrefetchGetRecordsCache.java create mode 100644 src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PreparedCheckpointer.java create mode 100644 src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/RecordsFetcherFactory.java delete mode 100644 src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownFuture.java create mode 100644 src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/SimpleRecordsFetcherFactory.java create mode 100644 src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/SynchronousGetRecordsRetrievalStrategy.java create mode 100644 src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/WorkerStateChangeListener.java create mode 100644 src/main/java/com/amazonaws/services/kinesis/multilang/messages/ShutdownRequestedMessage.java create mode 100644 src/test/java/com/amazonaws/services/dynamodbv2/streamsadapter/AmazonDynamoDBStreamsAdapterClient.java create mode 100644 src/test/java/com/amazonaws/services/dynamodbv2/streamsadapter/AmazonDynamoDBStreamsAdapterClientChild.java create mode 100644 src/test/java/com/amazonaws/services/kinesis/clientlibrary/config/DatePropertyValueDecoderTest.java create mode 100644 src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/AsynchronousGetRecordsRetrievalStrategyIntegrationTest.java create mode 100644 src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/AsynchronousGetRecordsRetrievalStrategyTest.java create mode 100644 src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/BlockingGetRecordsCacheTest.java create mode 100644 src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/GracefulShutdownCoordinatorTest.java create mode 100644 src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PrefetchGetRecordsCacheIntegrationTest.java create mode 100644 src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PrefetchGetRecordsCacheTest.java create mode 100644 src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PreparedCheckpointerTest.java create mode 100644 src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/RecordsFetcherFactoryTest.java delete mode 100644 src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownFutureTest.java rename src/test/{java => resources}/log4j.properties (100%) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..6bdaa999 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,6 @@ +*Issue #, if available:* + +*Description of changes:* + + +By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice. diff --git a/.travis.yml b/.travis.yml index ebb7a2ac..320f811c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: java jdk: - - openjdk7 - - oraclejdk7 + - openjdk8 - oraclejdk8 -sudo: false \ No newline at end of file +sudo: false +dist: trusty \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..56d1d436 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,329 @@ +# Changelog + +## Release 1.9.3 (October 30, 2018) +* Upgraded Apache Commons Lang from 2.6 to 3.7. + * [Issue #370](https://github.com/awslabs/amazon-kinesis-client/issues/370) + * [PR #406](https://github.com/awslabs/amazon-kinesis-client/pull/406) +* Upgraded Google Guava from 10.0 to 26.0-jre. + * [Issue #416](https://github.com/awslabs/amazon-kinesis-client/issues/416) + * [PR #421](https://github.com/awslabs/amazon-kinesis-client/pull/421) + +## Release 1.9.2 (September 4, 2018) +* Allow use of Immutable Clients + * [Issue #280](https://github.com/awslabs/amazon-kinesis-client/issues/280) + * [PR #305](https://github.com/awslabs/amazon-kinesis-client/pull/305) +* Allow the use of `AT_TIMESTAMP` for MultiLang Daemon Clients. + * [Issue #341](https://github.com/awslabs/amazon-kinesis-client/issues/341) + * [PR #342](https://github.com/awslabs/amazon-kinesis-client/pull/342) +* Update the cache for `KinesisProxy#getShard` on cache misses. + * [PR #344](https://github.com/awslabs/amazon-kinesis-client/pull/344) +* Changed release process to use a standard process. + * [PR #389](https://github.com/awslabs/amazon-kinesis-client/pull/389) +* Removed tests that expected a null region response for unknown regions. + * [PR #346](https://github.com/awslabs/amazon-kinesis-client/pull/346) +* Updated the version of the AWS Java SDK to 1.11.400 + +## Release 1.9.1 (April 30, 2018) +* Added the ability to create a prepared checkpoint when at `SHARD_END`. + * [PR #301](https://github.com/awslabs/amazon-kinesis-client/pull/301) +* Added the ability to subscribe to worker state change events. + * [PR #291](https://github.com/awslabs/amazon-kinesis-client/pull/291) +* Added support for custom lease managers. + A custom `LeaseManager` can be provided to `Worker.Builder` that will be used to provide lease services. + This makes it possible to implement custom lease management systems in addition to the default DynamoDB system. + * [PR #297](https://github.com/awslabs/amazon-kinesis-client/pull/297) +* Updated the version of the AWS Java SDK to 1.11.219 + +## Release 1.9.0 (February 6, 2018) +* Introducing support for ListShards API. This API is used in place of DescribeStream API to provide more throughput during ShardSyncTask. Please consult the [AWS Documentation for ListShards](https://docs.aws.amazon.com/kinesis/latest/APIReference/API_ListShards.html) for more information. + * ListShards supports higher call rate, which should reduce instances of throttling when attempting to synchronize the shard list. + * __WARNING: `ListShards` is a new API, and may require updating any explicit IAM policies__ + * Added configuration parameters for ListShards usage + + | Name | Default | Description | + | ---- | ------- | ----------- | + | [listShardsBackoffTimeInMillis](https://github.com/awslabs/amazon-kinesis-client/blob/3ae916c5fcdccd6b835c86ba7f6f53dd5b4c8b04/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java#L1412) | 1500 ms | This is the default backoff time between 2 ListShards calls when throttled. | + | [listShardsRetryAttempts](https://github.com/awslabs/amazon-kinesis-client/blob/3ae916c5fcdccd6b835c86ba7f6f53dd5b4c8b04/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java#L1423) | 50 | This is the maximum number of times the KinesisProxy will retry to make ListShards calls on being throttled. | + +* Updating the version of AWS Java SDK to 1.11.272. + * Version 1.11.272 is now the minimum support version of the SDK. +* Deprecating the following methods, and classes. These methods, and classes will be removed in a future release. + * Deprecated [IKinesisProxy#getStreamInfo](https://github.com/awslabs/amazon-kinesis-client/blob/3ae916c5fcdccd6b835c86ba7f6f53dd5b4c8b04/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/IKinesisProxy.java#L48-L62). + * Deprecated [IKinesisProxyFactory](https://github.com/awslabs/amazon-kinesis-client/blob/3ae916c5fcdccd6b835c86ba7f6f53dd5b4c8b04/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/IKinesisProxyFactory.java). + * Deprecated [KinesisProxyFactory](https://github.com/awslabs/amazon-kinesis-client/blob/3ae916c5fcdccd6b835c86ba7f6f53dd5b4c8b04/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxyFactory.java). + * Deprecated certain [KinesisProxy](https://github.com/awslabs/amazon-kinesis-client/blob/3ae916c5fcdccd6b835c86ba7f6f53dd5b4c8b04/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxy.java) constructors. + * [PR #293](https://github.com/awslabs/amazon-kinesis-client/pull/293) + +## Release 1.8.10 +* Allow providing a custom IKinesisProxy implementation. + * [PR #274](https://github.com/awslabs/amazon-kinesis-client/pull/274) +* Checkpointing on a different thread should no longer emit a warning about NullMetricsScope. + * [PR #284](https://github.com/awslabs/amazon-kinesis-client/pull/284) + * [Issue #48](https://github.com/awslabs/amazon-kinesis-client/issues/48) +* Upgraded the AWS Java SDK to version 1.11.271 + * [PR #287](https://github.com/awslabs/amazon-kinesis-client/pull/287) + +## Release 1.8.9 +* Allow disabling check for the case where a child shard has an open parent shard. + There is a race condition where it's possible for the a parent shard to appear open, while having child shards. This check can now be disabled by setting [`ignoreUnexpectedChildShards`](https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java#L1037) to true. + * [PR #240](https://github.com/awslabs/amazon-kinesis-client/pull/240) + * [Issue #210](https://github.com/awslabs/amazon-kinesis-client/issues/210) +* Upgraded the AWS SDK for Java to 1.11.261 + * [PR #281](https://github.com/awslabs/amazon-kinesis-client/pull/281) + +## Release 1.8.8 +* Fixed issues with leases losses due to `ExpiredIteratorException` in `PrefetchGetRecordsCache` and `AsynchronousFetchingStrategy`. + PrefetchGetRecordsCache will request for a new iterator and start fetching data again. + * [PR#263](https://github.com/awslabs/amazon-kinesis-client/pull/263) +* Added warning message for long running tasks. + Logging long running tasks can be enabled by setting the following configuration property: + + | Name | Default | Description | + | ---- | ------- | ----------- | + | [`logWarningForTaskAfterMillis`](https://github.com/awslabs/amazon-kinesis-client/blob/3de901ea9327370ed732af86c4d4999c8d99541c/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java#L1367) | Not set | Milliseconds after which the logger will log a warning message for the long running task | + + * [PR#259](https://github.com/awslabs/amazon-kinesis-client/pull/259) +* Handling spurious lease renewal failures gracefully. + Added better handling of DynamoDB failures when updating leases. These failures would occur when a request to DynamoDB appeared to fail, but was actually successful. + * [PR#247](https://github.com/awslabs/amazon-kinesis-client/pull/247) +* ShutdownTask gets retried if the previous attempt on the ShutdownTask fails. + * [PR#267](https://github.com/awslabs/amazon-kinesis-client/pull/267) +* Fix for using maxRecords from `KinesisClientLibConfiguration` in `GetRecordsCache` for fetching records. + * [PR#264](https://github.com/awslabs/amazon-kinesis-client/pull/264) + +## Release 1.8.7 +* Don't add a delay for synchronous requests to Kinesis + Removes a delay that had been added for synchronous `GetRecords` calls to Kinesis. + * [PR #256](https://github.com/awslabs/amazon-kinesis-client/pull/256) + +## Release 1.8.6 +* Add prefetching of records from Kinesis + Prefetching will retrieve and queue additional records from Kinesis while the application is processing existing records. + Prefetching can be enabled by setting [`dataFetchingStrategy`](https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java#L1317) to `PREFETCH_CACHED`. Once enabled an additional fetching thread will be started to retrieve records from Kinesis. Retrieved records will be held in a queue until the application is ready to process them. + Pre-fetching supports the following configuration values: + + | Name | Default | Description | + | ---- | ------- | ----------- | + | [`dataFetchingStrategy`](https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java#L1317) | `DEFAULT` | Which data fetching strategy to use | + | [`maxPendingProcessRecordsInput`](https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java#L1296) | 3 | The maximum number of process records input that can be queued | + | [`maxCacheByteSize`](https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java#L1307) | 8 MiB | The maximum number of bytes that can be queued | + | [`maxRecordsCount`](https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java#L1326) | 30,000 | The maximum number of records that can be queued | + | [`idleMillisBetweenCalls`](https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java#L1353) | 1,500 ms | The amount of time to wait between calls to Kinesis | + + * [PR #246](https://github.com/awslabs/amazon-kinesis-client/pull/246) + +## Release 1.8.5 (September 26, 2017) +* Only advance the shard iterator for the accepted response. + This fixes a race condition in the `KinesisDataFetcher` when it's being used to make asynchronous requests. The shard iterator is now only advanced when the retriever calls `DataFetcherResult#accept()`. + * [PR #230](https://github.com/awslabs/amazon-kinesis-client/pull/230) + * [Issue #231](https://github.com/awslabs/amazon-kinesis-client/issues/231) + +## Release 1.8.4 (September 22, 2017) +* Create a new completion service for each request. + This ensures that canceled tasks are discarded. This will prevent a cancellation exception causing issues processing records. + * [PR #227](https://github.com/awslabs/amazon-kinesis-client/pull/227) + * [Issue #226](https://github.com/awslabs/amazon-kinesis-client/issues/226) + +## Release 1.8.3 (September 22, 2017) +* Call shutdown on the retriever when the record processor is being shutdown + This fixes a bug that could leak threads if using the [`AsynchronousGetRecordsRetrievalStrategy`](https://github.com/awslabs/amazon-kinesis-client/blob/9a82b6bd05b3c9c5f8581af007141fa6d5f0fc4e/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/AsynchronousGetRecordsRetrievalStrategy.java#L42) is being used. + The asynchronous retriever is only used when [`KinesisClientLibConfiguration#retryGetRecordsInSeconds`](https://github.com/awslabs/amazon-kinesis-client/blob/01d2688bc6e68fd3fe5cb698cb65299d67ac930d/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java#L227), and [`KinesisClientLibConfiguration#maxGetRecordsThreadPool`](https://github.com/awslabs/amazon-kinesis-client/blob/01d2688bc6e68fd3fe5cb698cb65299d67ac930d/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java#L230) are set. + * [PR #222](https://github.com/awslabs/amazon-kinesis-client/pull/222) + +## Release 1.8.2 (September 20, 2017) +* Add support for two phase checkpoints + Applications can now set a pending checkpoint, before completing the checkpoint operation. Once the application has completed its checkpoint steps, the final checkpoint will clear the pending checkpoint. + Should the checkpoint fail the attempted sequence number is provided in the [`InitializationInput#getPendingCheckpointSequenceNumber`](https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/clientlibrary/types/InitializationInput.java#L81) otherwise the value will be null. + * [PR #188](https://github.com/awslabs/amazon-kinesis-client/pull/188) +* Support timeouts, and retry for GetRecords calls. + Applications can now set timeouts for GetRecord calls to Kinesis. As part of setting the timeout, the application must also provide a thread pool size for concurrent requests. + * [PR #214](https://github.com/awslabs/amazon-kinesis-client/pull/214) +* Notification when the lease table is throttled + When writes, or reads, to the lease table are throttled a warning will be emitted. If you're seeing this warning you should increase the IOPs for your lease table to prevent processing delays. + * [PR #212](https://github.com/awslabs/amazon-kinesis-client/pull/212) +* Support configuring the graceful shutdown timeout for MultiLang Clients + This adds support for setting the timeout that the Java process will wait for the MutliLang client to complete graceful shutdown. The timeout can be configured by adding `shutdownGraceMillis` to the properties file set to the number of milliseconds to wait. + * [PR #204](https://github.com/awslabs/amazon-kinesis-client/pull/204) + +## Release 1.8.1 (August 2, 2017) +* Support timeouts for calls to the MultiLang Daemon + This adds support for setting a timeout when dispatching records to the client record processor. If the record processor doesn't respond within the timeout the parent Java process will be terminated. This is a temporary fix to handle cases where the KCL becomes blocked while waiting for a client record processor. + The timeout for the this can be set by adding `timeoutInSeconds = `. The default for this is no timeout. + __Setting this can cause the KCL to exit suddenly, before using this ensure that you have an automated restart for your application__ + * [PR #195](https://github.com/awslabs/amazon-kinesis-client/pull/195) + * [Issue #185](https://github.com/awslabs/amazon-kinesis-client/issues/185) + +## Release 1.8.0 (July 25, 2017) +* Execute graceful shutdown on its own thread + * [PR #191](https://github.com/awslabs/amazon-kinesis-client/pull/191) + * [Issue #167](https://github.com/awslabs/amazon-kinesis-client/issues/167) +* Added support for controlling the size of the lease renewer thread pool + * [PR #177](https://github.com/awslabs/amazon-kinesis-client/pull/177) + * [Issue #171](https://github.com/awslabs/amazon-kinesis-client/issues/171) +* Require Java 8 and later + __Java 8 is now required for versions 1.8.0 of the amazon-kinesis-client and later.__ + * [PR #176](https://github.com/awslabs/amazon-kinesis-client/issues/176) + +## Release 1.7.6 (June 21, 2017) +* Added support for graceful shutdown in MultiLang Clients + * [PR #174](https://github.com/awslabs/amazon-kinesis-client/pull/174) + * [PR #182](https://github.com/awslabs/amazon-kinesis-client/pull/182) +* Updated documentation for `v2.IRecordProcessor#shutdown`, and `KinesisClientLibConfiguration#idleTimeBetweenReadsMillis` + * [PR #170](https://github.com/awslabs/amazon-kinesis-client/pull/170) +* Updated to version 1.11.151 of the AWS Java SDK + * [PR #183](https://github.com/awslabs/amazon-kinesis-client/pull/183) + +## Release 1.7.5 (April 7, 2017) +* Correctly handle throttling for DescribeStream, and save accumulated progress from individual calls. + * [PR #152](https://github.com/awslabs/amazon-kinesis-client/pull/152) +* Upgrade to version 1.11.115 of the AWS Java SDK + * [PR #155](https://github.com/awslabs/amazon-kinesis-client/pull/155) + +## Release 1.7.4 (February 27, 2017) +* Fixed an issue building JavaDoc for Java 8. + * [Issue #18](https://github.com/awslabs/amazon-kinesis-client/issues/18) + * [PR #141](https://github.com/awslabs/amazon-kinesis-client/pull/141) +* Reduce Throttling Messages to WARN, unless throttling occurs 6 times consecutively. + * [Issue #4](https://github.com/awslabs/amazon-kinesis-client/issues/4) + * [PR #140](https://github.com/awslabs/amazon-kinesis-client/pull/140) +* Fixed two bugs occurring in requestShutdown. + * Fixed a bug that prevented the worker from shutting down, via requestShutdown, when no leases were held. + * [Issue #128](https://github.com/awslabs/amazon-kinesis-client/issues/128) + * Fixed a bug that could trigger a NullPointerException if leases changed during requestShutdown. + * [Issue #129](https://github.com/awslabs/amazon-kinesis-client/issues/129) + * [PR #139](https://github.com/awslabs/amazon-kinesis-client/pull/139) +* Upgraded the AWS SDK Version to 1.11.91 + * [PR #138](https://github.com/awslabs/amazon-kinesis-client/pull/138) +* Use an executor returned from `ExecutorService.newFixedThreadPool` instead of constructing it by hand. + * [PR #135](https://github.com/awslabs/amazon-kinesis-client/pull/135) +* Correctly initialize DynamoDB client, when endpoint is explicitly set. + * [PR #142](https://github.com/awslabs/amazon-kinesis-client/pull/142) + +## Release 1.7.3 (January 9, 2017) +* Upgrade to the newest AWS Java SDK. + * [Amazon Kinesis Client Issue #27](https://github.com/awslabs/amazon-kinesis-client-python/issues/27) + * [PR #126](https://github.com/awslabs/amazon-kinesis-client/pull/126) + * [PR #125](https://github.com/awslabs/amazon-kinesis-client/pull/125) +* Added a direct dependency on commons-logging. + * [Issue #123](https://github.com/awslabs/amazon-kinesis-client/issues/123) + * [PR #124](https://github.com/awslabs/amazon-kinesis-client/pull/124) +* Make ShardInfo public to allow for custom ShardPrioritization strategies. + * [Issue #120](https://github.com/awslabs/amazon-kinesis-client/issues/120) + * [PR #127](https://github.com/awslabs/amazon-kinesis-client/pull/127) + +## Release 1.7.2 (November 7, 2016) +* MultiLangDaemon Feature Updates + The MultiLangDaemon has been upgraded to use the v2 interfaces, which allows access to enhanced checkpointing, and more information during record processor initialization. The MultiLangDaemon clients must be updated before they can take advantage of these new features. + +## Release 1.7.1 (November 3, 2016) +* General + * Allow disabling shard synchronization at startup. + * Applications can disable shard synchronization at startup. Disabling shard synchronization can application startup times for very large streams. + * [PR #102](https://github.com/awslabs/amazon-kinesis-client/pull/102) + * Applications can now request a graceful shutdown, and record processors that implement the IShutdownNotificationAware will be given a chance to checkpoint before being shutdown. + * This adds a [new interface](https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/clientlibrary/interfaces/v2/IShutdownNotificationAware.java), and a [new method on Worker](https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java#L539). + * [PR #109](https://github.com/awslabs/amazon-kinesis-client/pull/109) + * Solves [Issue #79](https://github.com/awslabs/amazon-kinesis-client/issues/79) +* MultiLangDaemon + * Applications can now use credential provides that accept string parameters. + * [PR #99](https://github.com/awslabs/amazon-kinesis-client/pull/99) + * Applications can now use different credentials for each service. + * [PR #111](https://github.com/awslabs/amazon-kinesis-client/pull/111) + +## Release 1.7.0 (August 22, 2016) +* Add support for time based iterators ([See GetShardIterator Documentation](http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetShardIterator.html)) + * [PR #94](https://github.com/awslabs/amazon-kinesis-client/pull/94) + The `KinesisClientLibConfiguration` now supports providing an initial time stamp position. + * This position is only used if there is no current checkpoint for the shard. + * This setting cannot be used with DynamoDB Streams + Resolves [Issue #88](https://github.com/awslabs/amazon-kinesis-client/issues/88) +* Allow Prioritization of Parent Shards for Task Assignment + * [PR #95](https://github.com/awslabs/amazon-kinesis-client/pull/95) + The `KinesisClientLibconfiguration` now supports providing a `ShardPrioritization` strategy. This strategy controls how the `Worker` determines which `ShardConsumer` to call next. This can improve processing for streams that split often, such as DynamoDB Streams. +* Remove direct dependency on `aws-java-sdk-core`, to allow independent versioning. + * [PR #92](https://github.com/awslabs/amazon-kinesis-client/pull/92) + **You may need to add a direct dependency on aws-java-sdk-core if other dependencies include an older version.** + +## Release 1.6.5 (July 25, 2016) +* Change LeaseManager to call DescribeTable before attempting to create the lease table. + * [Issue #36](https://github.com/awslabs/amazon-kinesis-client/issues/36) + * [PR #41](https://github.com/awslabs/amazon-kinesis-client/pull/41) + * [PR #67](https://github.com/awslabs/amazon-kinesis-client/pull/67) +* Allow DynamoDB lease table name to be specified + * [PR #61](https://github.com/awslabs/amazon-kinesis-client/pull/61) +* Add approximateArrivalTimestamp for JsonFriendlyRecord + * [PR #86](https://github.com/awslabs/amazon-kinesis-client/pull/86) +* Shutdown lease renewal thread pool on exit. + * [PR #84](https://github.com/awslabs/amazon-kinesis-client/pull/84) +* Wait for CloudWatch publishing thread to finish before exiting. + * [PR #82](https://github.com/awslabs/amazon-kinesis-client/pull/82) +* Added unit, and integration tests for the library. + +## Release 1.6.4 (July 6, 2016) +* Upgrade to AWS SDK for Java 1.11.14 + * [Issue #74](https://github.com/awslabs/amazon-kinesis-client/issues/74) + * [Issue #73](https://github.com/awslabs/amazon-kinesis-client/issues/73) +* **Maven Artifact Signing Change** + * Artifacts are now signed by the identity `Amazon Kinesis Tools ` + +## Release 1.6.3 (May 12, 2016) +* Fix format exception caused by DEBUG log in LeaseTaker [Issue # 68](https://github.com/awslabs/amazon-kinesis-client/issues/68) + +## Release 1.6.2 (March 23, 2016) +* Support for specifying max leases per worker and max leases to steal at a time. +* Support for specifying initial DynamoDB table read and write capacity. +* Support for parallel lease renewal. +* Support for graceful worker shutdown. +* Change DefaultCWMetricsPublisher log level to debug. [PR # 49](https://github.com/awslabs/amazon-kinesis-client/pull/49) +* Avoid NPE in MLD record processor shutdown if record processor was not initialized. [Issue # 29](https://github.com/awslabs/amazon-kinesis-client/issues/29) + +## Release 1.6.1 (September 23, 2015) +* Expose [approximateArrivalTimestamp](http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetRecords.html) for Records in processRecords API call. + +## Release 1.6.0 (July 31, 2015) +* Restores compatibility with [dynamodb-streams-kinesis-adapter](https://github.com/awslabs/dynamodb-streams-kinesis-adapter) (which was broken in 1.4.0). + +## Release 1.5.1 (July 20, 2015) +* KCL maven artifact 1.5.0 does not work with JDK 7. This release addresses this issue. + +## Release 1.5.0 (July 9, 2015) +* **[Metrics Enhancements][kinesis-guide-monitoring-with-kcl]** + * Support metrics level and dimension configurations to control CloudWatch metrics emitted by the KCL. + * Add new metrics that track time spent in record processor methods. + * Disable WorkerIdentifier dimension by default. +* **Exception Reporting** — Do not silently ignore exceptions in ShardConsumer. +* **AWS SDK Component Dependencies** — Depend only on AWS SDK components that are used. + +## Release 1.4.0 (June 2, 2015) +* Integration with the **[Kinesis Producer Library (KPL)][kinesis-guide-kpl]** + * Automatically de-aggregate records put into the Kinesis stream using the KPL. + * Support checkpointing at the individual user record level when multiple user records are aggregated into one Kinesis record using the KPL. + + See [Consumer De-aggregation with the KCL][kinesis-guide-consumer-deaggregation] for details. + +## Release 1.3.0 (May 22, 2015) +* A new metric called "MillisBehindLatest", which tracks how far consumers are from real time, is now uploaded to CloudWatch. + +## Release 1.2.1 (January 26, 2015) +* **MultiLangDaemon** — Changes to the MultiLangDaemon to make it easier to provide a custom worker. + +## Release 1.2 (October 21, 2014) +* **Multi-Language Support** — Amazon KCL now supports implementing record processors in any language by communicating with the daemon over [STDIN and STDOUT][multi-lang-protocol]. Python developers can directly use the [Amazon Kinesis Client Library for Python][kclpy] to write their data processing applications. + +## Release 1.1 (June 30, 2014) +* **Checkpointing at a specific sequence number** — The IRecordProcessorCheckpointer interface now supports checkpointing at a sequence number specified by the record processor. +* **Set region** — KinesisClientLibConfiguration now supports setting the region name to indicate the location of the Amazon Kinesis service. The Amazon DynamoDB table and Amazon CloudWatch metrics associated with your application will also use this region setting. + +[kinesis]: http://aws.amazon.com/kinesis +[kinesis-forum]: http://developer.amazonwebservices.com/connect/forum.jspa?forumID=169 +[kinesis-client-library-issues]: https://github.com/awslabs/amazon-kinesis-client/issues +[docs-signup]: http://docs.aws.amazon.com/AWSSdkDocsJava/latest/DeveloperGuide/java-dg-setup.html +[kinesis-guide]: http://docs.aws.amazon.com/kinesis/latest/dev/introduction.html +[kinesis-guide-begin]: http://docs.aws.amazon.com/kinesis/latest/dev/before-you-begin.html +[kinesis-guide-create]: http://docs.aws.amazon.com/kinesis/latest/dev/step-one-create-stream.html +[kinesis-guide-applications]: http://docs.aws.amazon.com/kinesis/latest/dev/kinesis-record-processor-app.html +[kinesis-guide-monitoring-with-kcl]: http://docs.aws.amazon.com//kinesis/latest/dev/monitoring-with-kcl.html +[kinesis-guide-kpl]: http://docs.aws.amazon.com//kinesis/latest/dev/developing-producers-with-kpl.html +[kinesis-guide-consumer-deaggregation]: http://docs.aws.amazon.com//kinesis/latest/dev/kinesis-kpl-consumer-deaggregation.html +[kclpy]: https://github.com/awslabs/amazon-kinesis-client-python +[multi-lang-protocol]: https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/multilang/package-info.java diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..3b644668 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,4 @@ +## Code of Conduct +This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). +For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact +opensource-codeofconduct@amazon.com with any additional questions or comments. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..ed9eb3e2 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,61 @@ +# Contributing Guidelines + +Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional +documentation, we greatly value feedback and contributions from our community. + +Please read through this document before submitting any issues or pull requests to ensure we have all the necessary +information to effectively respond to your bug report or contribution. + + +## Reporting Bugs/Feature Requests + +We welcome you to use the GitHub issue tracker to report bugs or suggest features. + +When filing an issue, please check [existing open](https://github.com/awslabs/amazon-kinesis-client/issues), or [recently closed](https://github.com/awslabs/amazon-kinesis-client/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20), issues to make sure somebody else hasn't already +reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: + +* A reproducible test case or series of steps +* The version of our code being used +* Any modifications you've made relevant to the bug +* Anything unusual about your environment or deployment + + +## Contributing via Pull Requests +Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: + +1. You are working against the latest source on the *master* branch. +2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. +3. You open an issue to discuss any significant work - we would hate for your time to be wasted. + +To send us a pull request, please: + +1. Fork the repository. +2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. +3. Ensure local tests pass. +4. Commit to your fork using clear commit messages. +5. Send us a pull request, answering any default questions in the pull request interface. +6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. + +GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and +[creating a pull request](https://help.github.com/articles/creating-a-pull-request/). + + +## Finding contributions to work on +Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels ((enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](https://github.com/awslabs/amazon-kinesis-client/labels/help%20wanted) issues is a great place to start. + + +## Code of Conduct +This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). +For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact +opensource-codeofconduct@amazon.com with any additional questions or comments. + + +## Security issue notifications +If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. + + +## Licensing + +See the [LICENSE](https://github.com/awslabs/amazon-kinesis-client/blob/master/LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. + +We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes. diff --git a/META-INF/MANIFEST.MF b/META-INF/MANIFEST.MF index dea3446e..b928a09f 100644 --- a/META-INF/MANIFEST.MF +++ b/META-INF/MANIFEST.MF @@ -2,9 +2,9 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: Amazon Kinesis Client Library for Java Bundle-SymbolicName: com.amazonaws.kinesisclientlibrary;singleton:=true -Bundle-Version: 1.7.4 +Bundle-Version: 1.9.1 Bundle-Vendor: Amazon Technologies, Inc -Bundle-RequiredExecutionEnvironment: JavaSE-1.7 +Bundle-RequiredExecutionEnvironment: JavaSE-1.8 Require-Bundle: org.apache.commons.codec;bundle-version="1.6", org.apache.commons.logging;bundle-version="1.1.3";visibility:=reexport, com.fasterxml.jackson.core.jackson-databind;bundle-version="2.5.3", @@ -12,7 +12,7 @@ Require-Bundle: org.apache.commons.codec;bundle-version="1.6", com.fasterxml.jackson.core.jackson-annotations;bundle-version="2.5.0", org.apache.httpcomponents.httpcore;bundle-version="4.3.3", org.apache.httpcomponents.httpclient;bundle-version="4.3.6" - com.amazonaws.sdk;bundle-version="1.11.14", + com.amazonaws.sdk;bundle-version="1.11.319", Export-Package: com.amazonaws.services.kinesis, com.amazonaws.services.kinesis.clientlibrary, com.amazonaws.services.kinesis.clientlibrary.config, diff --git a/README.md b/README.md index ba784a1f..d3fe6b72 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ -# Amazon Kinesis Client Library for Java [![Build Status](https://travis-ci.org/awslabs/amazon-kinesis-client.svg?branch=master)](https://travis-ci.org/awslabs/amazon-kinesis-client) +# Amazon Kinesis Client Library for Java +[![Build Status](https://travis-ci.org/awslabs/amazon-kinesis-client.svg?branch=master)](https://travis-ci.org/awslabs/amazon-kinesis-client) ![BuildStatus](https://codebuild.us-west-2.amazonaws.com/badges?uuid=eyJlbmNyeXB0ZWREYXRhIjoiaWo4bDYyUkpWaG9ZTy9zeFVoaVlWbEwxazdicDJLcmZwUUpFWVVBM0ZueEJSeFIzNkhURzdVbUd6WUZHcGNxa3BEUzNrL0I5Nzc4NE9rbXhvdEpNdlFRPSIsIml2UGFyYW1ldGVyU3BlYyI6IlZDaVZJSTM1QW95bFRTQnYiLCJtYXRlcmlhbFNldFNlcmlhbCI6MX0%3D&branch=v1.x) The **Amazon Kinesis Client Library for Java** (Amazon KCL) enables Java developers to easily consume and process data from [Amazon Kinesis][kinesis]. @@ -15,7 +16,7 @@ The **Amazon Kinesis Client Library for Java** (Amazon KCL) enables Java develop 1. **Sign up for AWS** — Before you begin, you need an AWS account. For more information about creating an AWS account and retrieving your AWS credentials, see [AWS Account and Credentials][docs-signup] in the AWS SDK for Java Developer Guide. 1. **Sign up for Amazon Kinesis** — Go to the Amazon Kinesis console to sign up for the service and create an Amazon Kinesis stream. For more information, see [Create an Amazon Kinesis Stream][kinesis-guide-create] in the Amazon Kinesis Developer Guide. -1. **Minimum requirements** — To use the Amazon Kinesis Client Library, you'll need **Java 1.7+**. For more information about Amazon Kinesis Client Library requirements, see [Before You Begin][kinesis-guide-begin] in the Amazon Kinesis Developer Guide. +1. **Minimum requirements** — To use the Amazon Kinesis Client Library, you'll need **Java 1.8+**. For more information about Amazon Kinesis Client Library requirements, see [Before You Begin][kinesis-guide-begin] in the Amazon Kinesis Developer Guide. 1. **Using the Amazon Kinesis Client Library** — The best way to get familiar with the Amazon Kinesis Client Library is to read [Developing Record Consumer Applications][kinesis-guide-applications] in the Amazon Kinesis Developer Guide. ## Building from Source @@ -23,146 +24,22 @@ The **Amazon Kinesis Client Library for Java** (Amazon KCL) enables Java develop After you've downloaded the code from GitHub, you can build it using Maven. To disable GPG signing in the build, use this command: `mvn clean install -Dgpg.skip=true` ## Integration with the Kinesis Producer Library -For producer-side developers using the **[Kinesis Producer Library (KPL)][kinesis-guide-kpl]**, the KCL integrates without additional effort.  When the KCL retrieves an aggregated Amazon Kinesis record consisting of multiple KPL user records, it will automatically invoke the KPL to extract the individual user records before returning them to the user. +For producer-side developers using the **[Kinesis Producer Library (KPL)][kinesis-guide-kpl]**, the KCL integrates without additional effort. When the KCL retrieves an aggregated Amazon Kinesis record consisting of multiple KPL user records, it will automatically invoke the KPL to extract the individual user records before returning them to the user. ## Amazon KCL support for other languages To make it easier for developers to write record processors in other languages, we have implemented a Java based daemon, called MultiLangDaemon that does all the heavy lifting. Our approach has the daemon spawn a sub-process, which in turn runs the record processor, which can be written in any language. The MultiLangDaemon process and the record processor sub-process communicate with each other over [STDIN and STDOUT using a defined protocol][multi-lang-protocol]. There will be a one to one correspondence amongst record processors, child processes, and shards. For Python developers specifically, we have abstracted these implementation details away and [expose an interface][kclpy] that enables you to focus on writing record processing logic in Python. This approach enables KCL to be language agnostic, while providing identical features and similar parallel processing model across all languages. ## Release Notes -### Release 1.7.4 (February 27, 2017) -* Fixed an issue building JavaDoc for Java 8. - * [Issue #18](https://github.com/awslabs/amazon-kinesis-client/issues/18) - * [PR #141](https://github.com/awslabs/amazon-kinesis-client/pull/141) -* Reduce Throttling Messages to WARN, unless throttling occurs 6 times consecutively. - * [Issue #4](https://github.com/awslabs/amazon-kinesis-client/issues/4) - * [PR #140](https://github.com/awslabs/amazon-kinesis-client/pull/140) -* Fixed two bugs occurring in requestShutdown. - * Fixed a bug that prevented the worker from shutting down, via requestShutdown, when no leases were held. - * [Issue #128](https://github.com/awslabs/amazon-kinesis-client/issues/128) - * Fixed a bug that could trigger a NullPointerException if leases changed during requestShutdown. - * [Issue #129](https://github.com/awslabs/amazon-kinesis-client/issues/129) - * [PR #139](https://github.com/awslabs/amazon-kinesis-client/pull/139) -* Upgraded the AWS SDK Version to 1.11.91 - * [PR #138](https://github.com/awslabs/amazon-kinesis-client/pull/138) -* Use an executor returned from `ExecutorService.newFixedThreadPool` instead of constructing it by hand. - * [PR #135](https://github.com/awslabs/amazon-kinesis-client/pull/135) -* Correctly initialize DynamoDB client, when endpoint is explicitly set. - * [PR #142](https://github.com/awslabs/amazon-kinesis-client/pull/142) -### Release 1.7.3 (January 9, 2017) -* Upgrade to the newest AWS Java SDK. - * [Amazon Kinesis Client Issue #27](https://github.com/awslabs/amazon-kinesis-client-python/issues/27) - * [PR #126](https://github.com/awslabs/amazon-kinesis-client/pull/126) - * [PR #125](https://github.com/awslabs/amazon-kinesis-client/pull/125) -* Added a direct dependency on commons-logging. - * [Issue #123](https://github.com/awslabs/amazon-kinesis-client/issues/123) - * [PR #124](https://github.com/awslabs/amazon-kinesis-client/pull/124) -* Make ShardInfo public to allow for custom ShardPrioritization strategies. - * [Issue #120](https://github.com/awslabs/amazon-kinesis-client/issues/120) - * [PR #127](https://github.com/awslabs/amazon-kinesis-client/pull/127) +### Latest Release (1.9.3 - October 30, 2018) +* Upgraded Apache Commons Lang from 2.6 to 3.7. + * [Issue #370](https://github.com/awslabs/amazon-kinesis-client/issues/370) + * [PR #406](https://github.com/awslabs/amazon-kinesis-client/pull/406) +* Upgraded Google Guava from 10.0 to 26.0-jre. + * [Issue #416](https://github.com/awslabs/amazon-kinesis-client/issues/416) + * [PR #421](https://github.com/awslabs/amazon-kinesis-client/pull/421) -### Release 1.7.2 (November 7, 2016) -* MultiLangDaemon Feature Updates - The MultiLangDaemon has been upgraded to use the v2 interfaces, which allows access to enhanced checkpointing, and more information during record processor initialization. The MultiLangDaemon clients must be updated before they can take advantage of these new features. - -### Release 1.7.1 (November 3, 2016) -* General - * Allow disabling shard synchronization at startup. - * Applications can disable shard synchronization at startup. Disabling shard synchronization can application startup times for very large streams. - * [PR #102](https://github.com/awslabs/amazon-kinesis-client/pull/102) - * Applications can now request a graceful shutdown, and record processors that implement the IShutdownNotificationAware will be given a chance to checkpoint before being shutdown. - * This adds a [new interface](https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/clientlibrary/interfaces/v2/IShutdownNotificationAware.java), and a [new method on Worker](https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java#L539). - * [PR #109](https://github.com/awslabs/amazon-kinesis-client/pull/109) - * Solves [Issue #79](https://github.com/awslabs/amazon-kinesis-client/issues/79) -* MultiLangDaemon - * Applications can now use credential provides that accept string parameters. - * [PR #99](https://github.com/awslabs/amazon-kinesis-client/pull/99) - * Applications can now use different credentials for each service. - * [PR #111](https://github.com/awslabs/amazon-kinesis-client/pull/111) - -### Release 1.7.0 (August 22, 2016) -* Add support for time based iterators ([See GetShardIterator Documentation](http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetShardIterator.html)) - * [PR #94](https://github.com/awslabs/amazon-kinesis-client/pull/94) - The `KinesisClientLibConfiguration` now supports providing an initial time stamp position. - * This position is only used if there is no current checkpoint for the shard. - * This setting cannot be used with DynamoDB Streams - Resolves [Issue #88](https://github.com/awslabs/amazon-kinesis-client/issues/88) -* Allow Prioritization of Parent Shards for Task Assignment - * [PR #95](https://github.com/awslabs/amazon-kinesis-client/pull/95) - The `KinesisClientLibconfiguration` now supports providing a `ShardPrioritization` strategy. This strategy controls how the `Worker` determines which `ShardConsumer` to call next. This can improve processing for streams that split often, such as DynamoDB Streams. -* Remove direct dependency on `aws-java-sdk-core`, to allow independent versioning. - * [PR #92](https://github.com/awslabs/amazon-kinesis-client/pull/92) - **You may need to add a direct dependency on aws-java-sdk-core if other dependencies include an older version.** - -### Release 1.6.5 (July 25, 2016) -* Change LeaseManager to call DescribeTable before attempting to create the lease table. - * [Issue #36](https://github.com/awslabs/amazon-kinesis-client/issues/36) - * [PR #41](https://github.com/awslabs/amazon-kinesis-client/pull/41) - * [PR #67](https://github.com/awslabs/amazon-kinesis-client/pull/67) -* Allow DynamoDB lease table name to be specified - * [PR #61](https://github.com/awslabs/amazon-kinesis-client/pull/61) -* Add approximateArrivalTimestamp for JsonFriendlyRecord - * [PR #86](https://github.com/awslabs/amazon-kinesis-client/pull/86) -* Shutdown lease renewal thread pool on exit. - * [PR #84](https://github.com/awslabs/amazon-kinesis-client/pull/84) -* Wait for CloudWatch publishing thread to finish before exiting. - * [PR #82](https://github.com/awslabs/amazon-kinesis-client/pull/82) -* Added unit, and integration tests for the library. - -### Release 1.6.4 (July 6, 2016) -* Upgrade to AWS SDK for Java 1.11.14 - * [Issue #74](https://github.com/awslabs/amazon-kinesis-client/issues/74) - * [Issue #73](https://github.com/awslabs/amazon-kinesis-client/issues/73) -* **Maven Artifact Signing Change** - * Artifacts are now signed by the identity `Amazon Kinesis Tools ` - -### Release 1.6.3 (May 12, 2016) -* Fix format exception caused by DEBUG log in LeaseTaker [Issue # 68](https://github.com/awslabs/amazon-kinesis-client/issues/68) - -### Release 1.6.2 (March 23, 2016) -* Support for specifying max leases per worker and max leases to steal at a time. -* Support for specifying initial DynamoDB table read and write capacity. -* Support for parallel lease renewal. -* Support for graceful worker shutdown. -* Change DefaultCWMetricsPublisher log level to debug. [PR # 49](https://github.com/awslabs/amazon-kinesis-client/pull/49) -* Avoid NPE in MLD record processor shutdown if record processor was not initialized. [Issue # 29](https://github.com/awslabs/amazon-kinesis-client/issues/29) - -### Release 1.6.1 (September 23, 2015) -* Expose [approximateArrivalTimestamp](http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetRecords.html) for Records in processRecords API call. - -### Release 1.6.0 (July 31, 2015) -* Restores compatibility with [dynamodb-streams-kinesis-adapter](https://github.com/awslabs/dynamodb-streams-kinesis-adapter) (which was broken in 1.4.0). - -### Release 1.5.1 (July 20, 2015) -* KCL maven artifact 1.5.0 does not work with JDK 7. This release addresses this issue. - -### Release 1.5.0 (July 9, 2015) -* **[Metrics Enhancements][kinesis-guide-monitoring-with-kcl]** - * Support metrics level and dimension configurations to control CloudWatch metrics emitted by the KCL. - * Add new metrics that track time spent in record processor methods. - * Disable WorkerIdentifier dimension by default. -* **Exception Reporting** — Do not silently ignore exceptions in ShardConsumer. -* **AWS SDK Component Dependencies** — Depend only on AWS SDK components that are used. - -### Release 1.4.0 (June 2, 2015) -* Integration with the **[Kinesis Producer Library (KPL)][kinesis-guide-kpl]** - * Automatically de-aggregate records put into the Kinesis stream using the KPL. - * Support checkpointing at the individual user record level when multiple user records are aggregated into one Kinesis record using the KPL. - - See [Consumer De-aggregation with the KCL][kinesis-guide-consumer-deaggregation] for details. - -### Release 1.3.0 (May 22, 2015) -* A new metric called "MillisBehindLatest", which tracks how far consumers are from real time, is now uploaded to CloudWatch. - -### Release 1.2.1 (January 26, 2015) -* **MultiLangDaemon** — Changes to the MultiLangDaemon to make it easier to provide a custom worker. - -### Release 1.2 (October 21, 2014) -* **Multi-Language Support** — Amazon KCL now supports implementing record processors in any language by communicating with the daemon over [STDIN and STDOUT][multi-lang-protocol]. Python developers can directly use the [Amazon Kinesis Client Library for Python][kclpy] to write their data processing applications. - -### Release 1.1 (June 30, 2014) -* **Checkpointing at a specific sequence number** — The IRecordProcessorCheckpointer interface now supports checkpointing at a sequence number specified by the record processor. -* **Set region** — KinesisClientLibConfiguration now supports setting the region name to indicate the location of the Amazon Kinesis service. The Amazon DynamoDB table and Amazon CloudWatch metrics associated with your application will also use this region setting. +### For remaining release notes check **[CHANGELOG.md][changelog-md]**. [kinesis]: http://aws.amazon.com/kinesis [kinesis-forum]: http://developer.amazonwebservices.com/connect/forum.jspa?forumID=169 @@ -177,4 +54,4 @@ To make it easier for developers to write record processors in other languages, [kinesis-guide-consumer-deaggregation]: http://docs.aws.amazon.com//kinesis/latest/dev/kinesis-kpl-consumer-deaggregation.html [kclpy]: https://github.com/awslabs/amazon-kinesis-client-python [multi-lang-protocol]: https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/multilang/package-info.java - +[changelog-md]: https://github.com/awslabs/amazon-kinesis-client/blob/master/CHANGELOG.md diff --git a/build.properties b/build.properties deleted file mode 100644 index 9a6b868a..00000000 --- a/build.properties +++ /dev/null @@ -1,10 +0,0 @@ -source.. = src/main/java,\ - src/main/resources -output.. = bin/ - -bin.includes = LICENSE.txt,\ - NOTICE.txt,\ - META-INF/,\ - . - -jre.compilation.profile = JavaSE-1.7 diff --git a/pom.xml b/pom.xml index 9503b971..995814b3 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ amazon-kinesis-client jar Amazon Kinesis Client Library for Java - 1.7.4 + 1.9.3 The Amazon Kinesis Client Library for Java enables Java developers to easily consume and process data from Amazon Kinesis. @@ -25,7 +25,7 @@ - 1.11.91 + 1.11.438 1.0.392 libsqlite4java ${project.build.directory}/test-lib @@ -50,7 +50,7 @@ com.google.guava guava - 18.0 + 26.0-jre com.google.protobuf @@ -58,9 +58,9 @@ 2.6.1 - commons-lang - commons-lang - 2.6 + org.apache.commons + commons-lang3 + 3.7 commons-logging @@ -99,16 +99,23 @@ com.amazonaws DynamoDBLocal - 1.11.0.1 + 1.11.86 + test + + + + log4j + log4j + 1.2.17 test - dynamodb-local - DynamoDB Local Release Repository - http://dynamodb-local.s3-website-us-west-2.amazonaws.com/release + dynamodblocal + AWS DynamoDB Local Release Repository + https://s3-us-west-2.amazonaws.com/dynamodb-local/release @@ -131,8 +138,8 @@ maven-compiler-plugin 3.2 - 1.7 - 1.7 + 1.8 + 1.8 UTF-8 @@ -242,9 +249,50 @@ + + org.apache.maven.plugins + maven-javadoc-plugin + 2.10.3 + + com.amazonaws.services.kinesis.producer.protobuf + + + + attach-javadocs + + jar + + + + + + org.apache.maven.plugins + maven-source-plugin + 3.0.1 + + + attach-sources + + jar + + + + + + + + sonatype-nexus-staging + https://oss.sonatype.org/content/repositories/snapshots + + + sonatype-nexus-staging + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + disable-java8-doclint @@ -255,6 +303,7 @@ -Xdoclint:none + publishing @@ -262,7 +311,7 @@ org.apache.maven.plugins maven-gpg-plugin - 1.5 + 1.6 sign-artifacts @@ -273,6 +322,17 @@ + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.8 + true + + sonatype-nexus-staging + https://oss.sonatype.org + false + + diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/config/DatePropertyValueDecorder.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/config/DatePropertyValueDecorder.java new file mode 100644 index 00000000..4f6a588c --- /dev/null +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/config/DatePropertyValueDecorder.java @@ -0,0 +1,53 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package com.amazonaws.services.kinesis.clientlibrary.config; + +import java.util.Arrays; +import java.util.Date; +import java.util.List; + +/** + * Provide Date property. + */ +class DatePropertyValueDecoder implements IPropertyValueDecoder { + + /** + * Constructor. + */ + DatePropertyValueDecoder() { + } + + /** + * @param value property value as String + * @return corresponding variable in correct type + */ + @Override + public Date decodeValue(String value) { + try { + return new Date(Long.parseLong(value) * 1000L); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Date property value must be numeric."); + } + } + + /** + * @return list of supported types + */ + @Override + public List> getSupportedTypes() { + return Arrays.asList(Date.class); + } + +} diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/config/KinesisClientLibConfigurator.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/config/KinesisClientLibConfigurator.java index e239f967..8059d6af 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/config/KinesisClientLibConfigurator.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/config/KinesisClientLibConfigurator.java @@ -66,6 +66,7 @@ public class KinesisClientLibConfigurator { Arrays.asList(new IntegerPropertyValueDecoder(), new LongPropertyValueDecoder(), new BooleanPropertyValueDecoder(), + new DatePropertyValueDecoder(), new AWSCredentialsProviderPropertyValueDecoder(), new StringPropertyValueDecoder(), new InitialPositionInStreamPropertyValueDecoder(), diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/interfaces/ICheckpoint.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/interfaces/ICheckpoint.java index d559bfc0..83c29b44 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/interfaces/ICheckpoint.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/interfaces/ICheckpoint.java @@ -15,6 +15,7 @@ package com.amazonaws.services.kinesis.clientlibrary.interfaces; import com.amazonaws.services.kinesis.clientlibrary.exceptions.KinesisClientLibException; +import com.amazonaws.services.kinesis.clientlibrary.lib.checkpoint.Checkpoint; import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber; /** @@ -46,4 +47,30 @@ public interface ICheckpoint { */ ExtendedSequenceNumber getCheckpoint(String shardId) throws KinesisClientLibException; + /** + * Get the current checkpoint stored for the specified shard, which holds the sequence numbers for the checkpoint + * and pending checkpoint. Useful for checking that the parent shard has been completely processed before we start + * processing the child shard. + * + * @param shardId Current checkpoint for this shard is fetched + * @return Current checkpoint object for this shard, null if there is no record for this shard. + * @throws KinesisClientLibException Thrown if we are unable to fetch the checkpoint + */ + Checkpoint getCheckpointObject(String shardId) throws KinesisClientLibException; + + + /** + * Record intent to checkpoint for a shard. Upon failover, the pendingCheckpointValue will be passed to the new + * RecordProcessor's initialize() method. + * + * @param shardId Checkpoint is specified for this shard. + * @param pendingCheckpoint Value of the pending checkpoint (e.g. Kinesis sequence number and subsequence number) + * @param concurrencyToken Used with conditional writes to prevent stale updates + * (e.g. if there was a fail over to a different record processor, we don't want to + * overwrite it's checkpoint) + * @throws KinesisClientLibException Thrown if we were unable to save the checkpoint + */ + void prepareCheckpoint(String shardId, ExtendedSequenceNumber pendingCheckpoint, String concurrencyToken) + throws KinesisClientLibException; + } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/interfaces/IPreparedCheckpointer.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/interfaces/IPreparedCheckpointer.java new file mode 100644 index 00000000..04827a63 --- /dev/null +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/interfaces/IPreparedCheckpointer.java @@ -0,0 +1,41 @@ +package com.amazonaws.services.kinesis.clientlibrary.interfaces; + +import com.amazonaws.services.kinesis.clientlibrary.exceptions.InvalidStateException; +import com.amazonaws.services.kinesis.clientlibrary.exceptions.KinesisClientLibDependencyException; +import com.amazonaws.services.kinesis.clientlibrary.exceptions.ShutdownException; +import com.amazonaws.services.kinesis.clientlibrary.exceptions.ThrottlingException; +import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber; + +/** + * Objects of this class are prepared to checkpoint at a specific sequence number. They use an + * IRecordProcessorCheckpointer to do the actual checkpointing, so their checkpoint is subject to the same 'didn't go + * backwards' validation as a normal checkpoint. + */ +public interface IPreparedCheckpointer { + + /** + * @return sequence number of pending checkpoint + */ + ExtendedSequenceNumber getPendingCheckpoint(); + + /** + * This method will record a pending checkpoint. + * + * @throws ThrottlingException Can't store checkpoint. Can be caused by checkpointing too frequently. + * Consider increasing the throughput/capacity of the checkpoint store or reducing checkpoint frequency. + * @throws ShutdownException The record processor instance has been shutdown. Another instance may have + * started processing some of these records already. + * The application should abort processing via this RecordProcessor instance. + * @throws InvalidStateException Can't store checkpoint. + * Unable to store the checkpoint in the DynamoDB table (e.g. table doesn't exist). + * @throws KinesisClientLibDependencyException Encountered an issue when storing the checkpoint. The application can + * backoff and retry. + * @throws IllegalArgumentException The sequence number being checkpointed is invalid because it is out of range, + * i.e. it is smaller than the last check point value (prepared or committed), or larger than the greatest + * sequence number seen by the associated record processor. + */ + void checkpoint() + throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException, + IllegalArgumentException; + +} \ No newline at end of file diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/interfaces/IRecordProcessorCheckpointer.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/interfaces/IRecordProcessorCheckpointer.java index f64d3c43..df4acc36 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/interfaces/IRecordProcessorCheckpointer.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/interfaces/IRecordProcessorCheckpointer.java @@ -120,4 +120,111 @@ public interface IRecordProcessorCheckpointer { void checkpoint(String sequenceNumber, long subSequenceNumber) throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException, IllegalArgumentException; + + /** + * This method will record a pending checkpoint at the last data record that was delivered to the record processor. + * If the application fails over between calling prepareCheckpoint() and checkpoint(), the init() method of the next + * IRecordProcessor for this shard will be informed of the prepared sequence number + * + * Application should use this to assist with idempotency across failover by calling prepareCheckpoint before having + * side effects, then by calling checkpoint on the returned PreparedCheckpointer after side effects are complete. + * Use the sequence number passed in to init() to behave idempotently. + * + * @return an IPreparedCheckpointer object that can be called later to persist the checkpoint. + * + * @throws ThrottlingException Can't store pending checkpoint. Can be caused by checkpointing too frequently. + * Consider increasing the throughput/capacity of the checkpoint store or reducing checkpoint frequency. + * @throws ShutdownException The record processor instance has been shutdown. Another instance may have + * started processing some of these records already. + * The application should abort processing via this RecordProcessor instance. + * @throws InvalidStateException Can't store pending checkpoint. + * Unable to store the checkpoint in the DynamoDB table (e.g. table doesn't exist). + * @throws KinesisClientLibDependencyException Encountered an issue when storing the pending checkpoint. The + * application can backoff and retry. + */ + IPreparedCheckpointer prepareCheckpoint() + throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException; + + /** + * This method will record a pending checkpoint at the at the provided record. This method is analogous to + * {@link #prepareCheckpoint()} but provides the ability to specify the record at which to prepare the checkpoint. + * + * @param record A record at which to prepare checkpoint in this shard. + * + * Application should use this to assist with idempotency across failover by calling prepareCheckpoint before having + * side effects, then by calling checkpoint on the returned PreparedCheckpointer after side effects are complete. + * Use the sequence number and application state passed in to init() to behave idempotently. + * + * @return an IPreparedCheckpointer object that can be called later to persist the checkpoint. + * + * @throws ThrottlingException Can't store pending checkpoint. Can be caused by checkpointing too frequently. + * Consider increasing the throughput/capacity of the checkpoint store or reducing checkpoint frequency. + * @throws ShutdownException The record processor instance has been shutdown. Another instance may have + * started processing some of these records already. + * The application should abort processing via this RecordProcessor instance. + * @throws InvalidStateException Can't store pending checkpoint. + * Unable to store the checkpoint in the DynamoDB table (e.g. table doesn't exist). + * @throws KinesisClientLibDependencyException Encountered an issue when storing the pending checkpoint. The + * application can backoff and retry. + * @throws IllegalArgumentException The sequence number is invalid for one of the following reasons: + * 1.) It appears to be out of range, i.e. it is smaller than the last check point value, or larger than the + * greatest sequence number seen by the associated record processor. + * 2.) It is not a valid sequence number for a record in this shard. + */ + IPreparedCheckpointer prepareCheckpoint(Record record) + throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException; + + /** + * This method will record a pending checkpoint at the provided sequenceNumber. This method is analogous to + * {@link #prepareCheckpoint()} but provides the ability to specify the sequence number at which to checkpoint. + * + * @param sequenceNumber A sequence number at which to prepare checkpoint in this shard. + + * @return an IPreparedCheckpointer object that can be called later to persist the checkpoint. + * + * @throws ThrottlingException Can't store pending checkpoint. Can be caused by checkpointing too frequently. + * Consider increasing the throughput/capacity of the checkpoint store or reducing checkpoint frequency. + * @throws ShutdownException The record processor instance has been shutdown. Another instance may have + * started processing some of these records already. + * The application should abort processing via this RecordProcessor instance. + * @throws InvalidStateException Can't store pending checkpoint. + * Unable to store the checkpoint in the DynamoDB table (e.g. table doesn't exist). + * @throws KinesisClientLibDependencyException Encountered an issue when storing the pending checkpoint. The + * application can backoff and retry. + * @throws IllegalArgumentException The sequence number is invalid for one of the following reasons: + * 1.) It appears to be out of range, i.e. it is smaller than the last check point value, or larger than the + * greatest sequence number seen by the associated record processor. + * 2.) It is not a valid sequence number for a record in this shard. + */ + IPreparedCheckpointer prepareCheckpoint(String sequenceNumber) + throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException, + IllegalArgumentException; + + /** + * This method will record a pending checkpoint at the provided sequenceNumber and subSequenceNumber, the latter for + * aggregated records produced with the Producer Library. This method is analogous to {@link #prepareCheckpoint()} + * but provides the ability to specify the sequence number at which to checkpoint + * + * @param sequenceNumber A sequence number at which to prepare checkpoint in this shard. + * @param subSequenceNumber A subsequence number at which to prepare checkpoint within this shard. + * + * @return an IPreparedCheckpointer object that can be called later to persist the checkpoint. + * + * @throws ThrottlingException Can't store pending checkpoint. Can be caused by checkpointing too frequently. + * Consider increasing the throughput/capacity of the checkpoint store or reducing checkpoint frequency. + * @throws ShutdownException The record processor instance has been shutdown. Another instance may have + * started processing some of these records already. + * The application should abort processing via this RecordProcessor instance. + * @throws InvalidStateException Can't store pending checkpoint. + * Unable to store the checkpoint in the DynamoDB table (e.g. table doesn't exist). + * @throws KinesisClientLibDependencyException Encountered an issue when storing the pending checkpoint. The + * application can backoff and retry. + * @throws IllegalArgumentException The sequence number is invalid for one of the following reasons: + * 1.) It appears to be out of range, i.e. it is smaller than the last check point value, or larger than the + * greatest sequence number seen by the associated record processor. + * 2.) It is not a valid sequence number for a record in this shard. + */ + IPreparedCheckpointer prepareCheckpoint(String sequenceNumber, long subSequenceNumber) + throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException, + IllegalArgumentException; } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/interfaces/v2/IRecordProcessor.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/interfaces/v2/IRecordProcessor.java index 25087f03..bd445ac9 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/interfaces/v2/IRecordProcessor.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/interfaces/v2/IRecordProcessor.java @@ -45,10 +45,16 @@ public interface IRecordProcessor { /** * Invoked by the Amazon Kinesis Client Library to indicate it will no longer send data records to this - * RecordProcessor instance. + * RecordProcessor instance. * - * @param shutdownInput Provides information and capabilities (eg checkpointing) related to shutdown of this record - * processor. + *

Warning

+ * + * When the value of {@link ShutdownInput#getShutdownReason()} is + * {@link com.amazonaws.services.kinesis.clientlibrary.lib.worker.ShutdownReason#TERMINATE} it is required that you + * checkpoint. Failure to do so will result in an IllegalArgumentException, and the KCL no longer making progress. + * + * @param shutdownInput + * Provides information and capabilities (eg checkpointing) related to shutdown of this record processor. */ void shutdown(ShutdownInput shutdownInput); diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/interfaces/v2/IShutdownNotificationAware.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/interfaces/v2/IShutdownNotificationAware.java index 82a18a0e..b4d4629c 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/interfaces/v2/IShutdownNotificationAware.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/interfaces/v2/IShutdownNotificationAware.java @@ -1,3 +1,17 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ package com.amazonaws.services.kinesis.clientlibrary.interfaces.v2; import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer; diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/Checkpoint.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/Checkpoint.java new file mode 100644 index 00000000..d81c632f --- /dev/null +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/Checkpoint.java @@ -0,0 +1,27 @@ +package com.amazonaws.services.kinesis.clientlibrary.lib.checkpoint; + +import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber; +import lombok.Data; + +/** + * A class encapsulating the 2 pieces of state stored in a checkpoint. + */ +@Data public class Checkpoint { + + private final ExtendedSequenceNumber checkpoint; + private final ExtendedSequenceNumber pendingCheckpoint; + + /** + * Constructor. + * + * @param checkpoint the checkpoint sequence number - cannot be null or empty. + * @param pendingCheckpoint the pending checkpoint sequence number - can be null. + */ + public Checkpoint(ExtendedSequenceNumber checkpoint, ExtendedSequenceNumber pendingCheckpoint) { + if (checkpoint == null || checkpoint.getSequenceNumber().isEmpty()) { + throw new IllegalArgumentException("Checkpoint cannot be null or empty"); + } + this.checkpoint = checkpoint; + this.pendingCheckpoint = pendingCheckpoint; + } +} diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/AsynchronousGetRecordsRetrievalStrategy.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/AsynchronousGetRecordsRetrievalStrategy.java new file mode 100644 index 00000000..2db74fba5 --- /dev/null +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/AsynchronousGetRecordsRetrievalStrategy.java @@ -0,0 +1,154 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package com.amazonaws.services.kinesis.clientlibrary.lib.worker; + +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletionService; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; + +import com.amazonaws.services.kinesis.metrics.impl.MetricsHelper; +import com.amazonaws.services.kinesis.metrics.impl.ThreadSafeMetricsDelegatingScope; +import com.amazonaws.services.kinesis.model.ExpiredIteratorException; +import com.amazonaws.services.kinesis.model.GetRecordsResult; +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +import lombok.NonNull; +import lombok.extern.apachecommons.CommonsLog; + +/** + * + */ +@CommonsLog +public class AsynchronousGetRecordsRetrievalStrategy implements GetRecordsRetrievalStrategy { + private static final int TIME_TO_KEEP_ALIVE = 5; + private static final int CORE_THREAD_POOL_COUNT = 1; + + private final KinesisDataFetcher dataFetcher; + private final ExecutorService executorService; + private final int retryGetRecordsInSeconds; + private final String shardId; + final Supplier> completionServiceSupplier; + + public AsynchronousGetRecordsRetrievalStrategy(@NonNull final KinesisDataFetcher dataFetcher, + final int retryGetRecordsInSeconds, final int maxGetRecordsThreadPool, String shardId) { + this(dataFetcher, buildExector(maxGetRecordsThreadPool, shardId), retryGetRecordsInSeconds, shardId); + } + + public AsynchronousGetRecordsRetrievalStrategy(final KinesisDataFetcher dataFetcher, + final ExecutorService executorService, final int retryGetRecordsInSeconds, String shardId) { + this(dataFetcher, executorService, retryGetRecordsInSeconds, () -> new ExecutorCompletionService<>(executorService), + shardId); + } + + AsynchronousGetRecordsRetrievalStrategy(KinesisDataFetcher dataFetcher, ExecutorService executorService, + int retryGetRecordsInSeconds, Supplier> completionServiceSupplier, + String shardId) { + this.dataFetcher = dataFetcher; + this.executorService = executorService; + this.retryGetRecordsInSeconds = retryGetRecordsInSeconds; + this.completionServiceSupplier = completionServiceSupplier; + this.shardId = shardId; + } + + @Override + public GetRecordsResult getRecords(final int maxRecords) { + if (executorService.isShutdown()) { + throw new IllegalStateException("Strategy has been shutdown"); + } + GetRecordsResult result = null; + CompletionService completionService = completionServiceSupplier.get(); + Set> futures = new HashSet<>(); + Callable retrieverCall = createRetrieverCallable(maxRecords); + try { + while (true) { + try { + futures.add(completionService.submit(retrieverCall)); + } catch (RejectedExecutionException e) { + log.warn("Out of resources, unable to start additional requests."); + } + + try { + Future resultFuture = completionService.poll(retryGetRecordsInSeconds, + TimeUnit.SECONDS); + if (resultFuture != null) { + // + // Fix to ensure that we only let the shard iterator advance when we intend to return the result + // to the caller. This ensures that the shard iterator is consistently advance in step with + // what the caller sees. + // + result = resultFuture.get().accept(); + break; + } + } catch (ExecutionException e) { + if (e.getCause() instanceof ExpiredIteratorException) { + throw (ExpiredIteratorException) e.getCause(); + } + log.error("ExecutionException thrown while trying to get records", e); + } catch (InterruptedException e) { + log.error("Thread was interrupted", e); + break; + } + } + } finally { + futures.forEach(f -> f.cancel(true)); + } + return result; + } + + private Callable createRetrieverCallable(int maxRecords) { + ThreadSafeMetricsDelegatingScope metricsScope = new ThreadSafeMetricsDelegatingScope(MetricsHelper.getMetricsScope()); + return () -> { + try { + MetricsHelper.setMetricsScope(metricsScope); + return dataFetcher.getRecords(maxRecords); + } finally { + MetricsHelper.unsetMetricsScope(); + } + }; + } + + @Override + public void shutdown() { + executorService.shutdownNow(); + } + + @Override + public boolean isShutdown() { + return executorService.isShutdown(); + } + + private static ExecutorService buildExector(int maxGetRecordsThreadPool, String shardId) { + String threadNameFormat = "get-records-worker-" + shardId + "-%d"; + return new ThreadPoolExecutor(CORE_THREAD_POOL_COUNT, maxGetRecordsThreadPool, TIME_TO_KEEP_ALIVE, + TimeUnit.SECONDS, new LinkedBlockingQueue<>(1), + new ThreadFactoryBuilder().setDaemon(true).setNameFormat(threadNameFormat).build(), + new ThreadPoolExecutor.AbortPolicy()); + } + + @Override + public KinesisDataFetcher getDataFetcher() { + return dataFetcher; + } +} diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/BlockingGetRecordsCache.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/BlockingGetRecordsCache.java new file mode 100644 index 00000000..021d886b --- /dev/null +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/BlockingGetRecordsCache.java @@ -0,0 +1,65 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazonaws.services.kinesis.clientlibrary.lib.worker; + +import java.time.Duration; +import java.time.Instant; + +import com.amazonaws.services.kinesis.clientlibrary.types.ProcessRecordsInput; +import com.amazonaws.services.kinesis.model.GetRecordsResult; + +import lombok.extern.apachecommons.CommonsLog; + +/** + * This is the BlockingGetRecordsCache class. This class blocks any calls to the getRecords on the + * GetRecordsRetrievalStrategy class. + */ +@CommonsLog +public class BlockingGetRecordsCache implements GetRecordsCache { + private final int maxRecordsPerCall; + private final GetRecordsRetrievalStrategy getRecordsRetrievalStrategy; + + public BlockingGetRecordsCache(final int maxRecordsPerCall, + final GetRecordsRetrievalStrategy getRecordsRetrievalStrategy) { + this.maxRecordsPerCall = maxRecordsPerCall; + this.getRecordsRetrievalStrategy = getRecordsRetrievalStrategy; + } + + @Override + public void start() { + // + // Nothing to do here + // + } + + @Override + public ProcessRecordsInput getNextResult() { + GetRecordsResult getRecordsResult = getRecordsRetrievalStrategy.getRecords(maxRecordsPerCall); + return new ProcessRecordsInput() + .withRecords(getRecordsResult.getRecords()) + .withMillisBehindLatest(getRecordsResult.getMillisBehindLatest()); + } + + @Override + public GetRecordsRetrievalStrategy getGetRecordsRetrievalStrategy() { + return getRecordsRetrievalStrategy; + } + + @Override + public void shutdown() { + getRecordsRetrievalStrategy.shutdown(); + } +} diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ConsumerStates.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ConsumerStates.java index 2d92d7d7..c0bdc060 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ConsumerStates.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ConsumerStates.java @@ -1,3 +1,17 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; /** @@ -237,9 +251,14 @@ class ConsumerStates { @Override public ITask createTask(ShardConsumer consumer) { - return new InitializeTask(consumer.getShardInfo(), consumer.getRecordProcessor(), consumer.getCheckpoint(), - consumer.getRecordProcessorCheckpointer(), consumer.getDataFetcher(), - consumer.getTaskBackoffTimeMillis(), consumer.getStreamConfig()); + return new InitializeTask(consumer.getShardInfo(), + consumer.getRecordProcessor(), + consumer.getCheckpoint(), + consumer.getRecordProcessorCheckpointer(), + consumer.getDataFetcher(), + consumer.getTaskBackoffTimeMillis(), + consumer.getStreamConfig(), + consumer.getGetRecordsCache()); } @Override @@ -293,9 +312,14 @@ class ConsumerStates { @Override public ITask createTask(ShardConsumer consumer) { - return new ProcessTask(consumer.getShardInfo(), consumer.getStreamConfig(), consumer.getRecordProcessor(), - consumer.getRecordProcessorCheckpointer(), consumer.getDataFetcher(), - consumer.getTaskBackoffTimeMillis(), consumer.isSkipShardSyncAtWorkerInitializationIfLeasesExist()); + return new ProcessTask(consumer.getShardInfo(), + consumer.getStreamConfig(), + consumer.getRecordProcessor(), + consumer.getRecordProcessorCheckpointer(), + consumer.getDataFetcher(), + consumer.getTaskBackoffTimeMillis(), + consumer.isSkipShardSyncAtWorkerInitializationIfLeasesExist(), + consumer.getGetRecordsCache()); } @Override @@ -354,8 +378,10 @@ class ConsumerStates { @Override public ITask createTask(ShardConsumer consumer) { - return new ShutdownNotificationTask(consumer.getRecordProcessor(), consumer.getRecordProcessorCheckpointer(), - consumer.getShutdownNotification(), consumer.getShardInfo()); + return new ShutdownNotificationTask(consumer.getRecordProcessor(), + consumer.getRecordProcessorCheckpointer(), + consumer.getShutdownNotification(), + consumer.getShardInfo()); } @Override @@ -494,12 +520,17 @@ class ConsumerStates { @Override public ITask createTask(ShardConsumer consumer) { - return new ShutdownTask(consumer.getShardInfo(), consumer.getRecordProcessor(), - consumer.getRecordProcessorCheckpointer(), consumer.getShutdownReason(), + return new ShutdownTask(consumer.getShardInfo(), + consumer.getRecordProcessor(), + consumer.getRecordProcessorCheckpointer(), + consumer.getShutdownReason(), consumer.getStreamConfig().getStreamProxy(), consumer.getStreamConfig().getInitialPositionInStream(), - consumer.isCleanupLeasesOfCompletedShards(), consumer.getLeaseManager(), - consumer.getTaskBackoffTimeMillis()); + consumer.isCleanupLeasesOfCompletedShards(), + consumer.isIgnoreUnexpectedChildShards(), + consumer.getLeaseManager(), + consumer.getTaskBackoffTimeMillis(), + consumer.getGetRecordsCache()); } @Override diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/DataFetcherResult.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/DataFetcherResult.java new file mode 100644 index 00000000..a7121ff2 --- /dev/null +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/DataFetcherResult.java @@ -0,0 +1,37 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. Licensed under the Amazon Software License + * (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at + * http://aws.amazon.com/asl/ or in the "license" file accompanying this file. This file is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific + * language governing permissions and limitations under the License. + */ +package com.amazonaws.services.kinesis.clientlibrary.lib.worker; + +import com.amazonaws.services.kinesis.model.GetRecordsResult; + +/** + * Represents the result from the DataFetcher, and allows the receiver to accept a result + */ +public interface DataFetcherResult { + /** + * The result of the request to Kinesis + * + * @return The result of the request, this can be null if the request failed. + */ + GetRecordsResult getResult(); + + /** + * Accepts the result, and advances the shard iterator. A result from the data fetcher must be accepted before any + * further progress can be made. + * + * @return the result of the request, this can be null if the request failed. + */ + GetRecordsResult accept(); + + /** + * Indicates whether this result is at the end of the shard or not + * + * @return true if the result is at the end of a shard, false otherwise + */ + boolean isShardEnd(); +} diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/DataFetchingStrategy.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/DataFetchingStrategy.java new file mode 100644 index 00000000..05c2ab3f --- /dev/null +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/DataFetchingStrategy.java @@ -0,0 +1,8 @@ +package com.amazonaws.services.kinesis.clientlibrary.lib.worker; + +/** + * + */ +public enum DataFetchingStrategy { + DEFAULT, PREFETCH_CACHED; +} diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/DoesNothingPreparedCheckpointer.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/DoesNothingPreparedCheckpointer.java new file mode 100644 index 00000000..d40f51d3 --- /dev/null +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/DoesNothingPreparedCheckpointer.java @@ -0,0 +1,66 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package com.amazonaws.services.kinesis.clientlibrary.lib.worker; + +import com.amazonaws.services.kinesis.clientlibrary.exceptions.InvalidStateException; +import com.amazonaws.services.kinesis.clientlibrary.exceptions.KinesisClientLibDependencyException; +import com.amazonaws.services.kinesis.clientlibrary.exceptions.ShutdownException; +import com.amazonaws.services.kinesis.clientlibrary.exceptions.ThrottlingException; +import com.amazonaws.services.kinesis.clientlibrary.interfaces.IPreparedCheckpointer; +import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber; + +/** + * A special IPreparedCheckpointer that does nothing, which can be used when preparing a checkpoint at the current + * checkpoint sequence number where it is never necessary to do another checkpoint. + * This simplifies programming by preventing application developers from having to reason about whether + * their application has processed records before calling prepareCheckpoint + * + * Here's why it's safe to do nothing: + * The only way to checkpoint at current checkpoint value is to have a record processor that gets + * initialized, processes 0 records, then calls prepareCheckpoint(). The value in the table is the same, so there's + * no reason to overwrite it with another copy of itself. + */ +public class DoesNothingPreparedCheckpointer implements IPreparedCheckpointer { + + private final ExtendedSequenceNumber sequenceNumber; + + /** + * Constructor. + * @param sequenceNumber the sequence number value + */ + public DoesNothingPreparedCheckpointer(ExtendedSequenceNumber sequenceNumber) { + this.sequenceNumber = sequenceNumber; + } + + /** + * {@inheritDoc} + */ + @Override + public ExtendedSequenceNumber getPendingCheckpoint() { + return sequenceNumber; + } + + /** + * {@inheritDoc} + */ + @Override + public void checkpoint() + throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException, + IllegalArgumentException { + // This method does nothing + } + +} + diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/GetRecordsCache.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/GetRecordsCache.java new file mode 100644 index 00000000..dba24f8d --- /dev/null +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/GetRecordsCache.java @@ -0,0 +1,43 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazonaws.services.kinesis.clientlibrary.lib.worker; + +import com.amazonaws.services.kinesis.clientlibrary.types.ProcessRecordsInput; + +/** + * This class is used as a cache for Prefetching data from Kinesis. + */ +public interface GetRecordsCache { + /** + * This method calls the start behavior on the cache, if available. + */ + void start(); + + /** + * This method returns the next set of records from the Cache if present, or blocks the request till it gets the + * next set of records back from Kinesis. + * + * @return The next set of records. + */ + ProcessRecordsInput getNextResult(); + + GetRecordsRetrievalStrategy getGetRecordsRetrievalStrategy(); + + /** + * This method calls the shutdown behavior on the cache, if available. + */ + void shutdown(); +} diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/GetRecordsRetrievalStrategy.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/GetRecordsRetrievalStrategy.java new file mode 100644 index 00000000..4f474887 --- /dev/null +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/GetRecordsRetrievalStrategy.java @@ -0,0 +1,54 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package com.amazonaws.services.kinesis.clientlibrary.lib.worker; + +import com.amazonaws.services.kinesis.model.GetRecordsResult; + +/** + * Represents a strategy to retrieve records from Kinesis. Allows for variations on how records are retrieved from + * Kinesis. + */ +public interface GetRecordsRetrievalStrategy { + /** + * Gets a set of records from Kinesis. + * + * @param maxRecords + * passed to Kinesis, and can be used to restrict the number of records returned from Kinesis. + * @return the resulting records. + * @throws IllegalStateException + * if the strategy has been shutdown. + */ + GetRecordsResult getRecords(int maxRecords); + + /** + * Releases any resources used by the strategy. Once the strategy is shutdown it is no longer safe to call + * {@link #getRecords(int)}. + */ + void shutdown(); + + /** + * Returns whether this strategy has been shutdown. + * + * @return true if the strategy has been shutdown, false otherwise. + */ + boolean isShutdown(); + + /** + * Returns the KinesisDataFetcher used to getRecords from Kinesis. + * + * @return KinesisDataFetcher + */ + KinesisDataFetcher getDataFetcher(); +} diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/GetRecordsRetriever.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/GetRecordsRetriever.java new file mode 100644 index 00000000..d5b4a782 --- /dev/null +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/GetRecordsRetriever.java @@ -0,0 +1,12 @@ +package com.amazonaws.services.kinesis.clientlibrary.lib.worker; + +import com.amazonaws.services.kinesis.model.GetRecordsResult; + +import java.util.concurrent.Callable; + +/** + * This class uses the GetRecordsRetrievalStrategy class to retrieve the next set of records and update the cache. + */ +public interface GetRecordsRetriever { + GetRecordsResult getNextRecords(int maxRecords); +} diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/GracefulShutdownContext.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/GracefulShutdownContext.java new file mode 100644 index 00000000..22a4d92b --- /dev/null +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/GracefulShutdownContext.java @@ -0,0 +1,33 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package com.amazonaws.services.kinesis.clientlibrary.lib.worker; + +import lombok.Data; + +import java.util.concurrent.CountDownLatch; + +@Data +class GracefulShutdownContext { + private final CountDownLatch shutdownCompleteLatch; + private final CountDownLatch notificationCompleteLatch; + private final Worker worker; + + static GracefulShutdownContext SHUTDOWN_ALREADY_COMPLETED = new GracefulShutdownContext(null, null, null); + + boolean isShutdownAlreadyCompleted() { + return shutdownCompleteLatch == null && notificationCompleteLatch == null && worker == null; + } + +} diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/GracefulShutdownCoordinator.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/GracefulShutdownCoordinator.java new file mode 100644 index 00000000..97bef9e3 --- /dev/null +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/GracefulShutdownCoordinator.java @@ -0,0 +1,163 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package com.amazonaws.services.kinesis.clientlibrary.lib.worker; + +import java.util.concurrent.Callable; +import java.util.concurrent.Future; +import java.util.concurrent.FutureTask; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +class GracefulShutdownCoordinator { + + Future startGracefulShutdown(Callable shutdownCallable) { + FutureTask task = new FutureTask<>(shutdownCallable); + Thread shutdownThread = new Thread(task, "RequestedShutdownThread"); + shutdownThread.start(); + return task; + + } + + Callable createGracefulShutdownCallable(Callable startWorkerShutdown) { + return new GracefulShutdownCallable(startWorkerShutdown); + } + + static class GracefulShutdownCallable implements Callable { + + private static final Log log = LogFactory.getLog(GracefulShutdownCallable.class); + + private final Callable startWorkerShutdown; + + GracefulShutdownCallable(Callable startWorkerShutdown) { + this.startWorkerShutdown = startWorkerShutdown; + } + + private boolean isWorkerShutdownComplete(GracefulShutdownContext context) { + return context.getWorker().isShutdownComplete() || context.getWorker().getShardInfoShardConsumerMap().isEmpty(); + } + + private String awaitingLogMessage(GracefulShutdownContext context) { + long awaitingNotification = context.getNotificationCompleteLatch().getCount(); + long awaitingFinalShutdown = context.getShutdownCompleteLatch().getCount(); + + return String.format( + "Waiting for %d record process to complete shutdown notification, and %d record processor to complete final shutdown ", + awaitingNotification, awaitingFinalShutdown); + } + + private String awaitingFinalShutdownMessage(GracefulShutdownContext context) { + long outstanding = context.getShutdownCompleteLatch().getCount(); + return String.format("Waiting for %d record processors to complete final shutdown", outstanding); + } + + private boolean waitForRecordProcessors(GracefulShutdownContext context) { + + // + // Awaiting for all ShardConsumer/RecordProcessors to be notified that a shutdown has been requested. + // There is the possibility of a race condition where a lease is terminated after the shutdown request + // notification is started, but before the ShardConsumer is sent the notification. In this case the + // ShardConsumer would start the lease loss shutdown, and may never call the notification methods. + // + try { + while (!context.getNotificationCompleteLatch().await(1, TimeUnit.SECONDS)) { + if (Thread.interrupted()) { + throw new InterruptedException(); + } + log.info(awaitingLogMessage(context)); + if (workerShutdownWithRemaining(context.getShutdownCompleteLatch().getCount(), context)) { + return false; + } + } + } catch (InterruptedException ie) { + log.warn("Interrupted while waiting for notification complete, terminating shutdown. " + + awaitingLogMessage(context)); + return false; + } + + if (Thread.interrupted()) { + log.warn("Interrupted before worker shutdown, terminating shutdown"); + return false; + } + + // + // Once all record processors have been notified of the shutdown it is safe to allow the worker to + // start its shutdown behavior. Once shutdown starts it will stop renewer, and drop any remaining leases. + // + context.getWorker().shutdown(); + + if (Thread.interrupted()) { + log.warn("Interrupted after worker shutdown, terminating shutdown"); + return false; + } + + // + // Want to wait for all the remaining ShardConsumers/RecordProcessor's to complete their final shutdown + // processing. This should really be a no-op since as part of the notification completion the lease for + // ShardConsumer is terminated. + // + try { + while (!context.getShutdownCompleteLatch().await(1, TimeUnit.SECONDS)) { + if (Thread.interrupted()) { + throw new InterruptedException(); + } + log.info(awaitingFinalShutdownMessage(context)); + if (workerShutdownWithRemaining(context.getShutdownCompleteLatch().getCount(), context)) { + return false; + } + } + } catch (InterruptedException ie) { + log.warn("Interrupted while waiting for shutdown completion, terminating shutdown. " + + awaitingFinalShutdownMessage(context)); + return false; + } + return true; + } + + /** + * This checks to see if the worker has already hit it's shutdown target, while there is outstanding record + * processors. This maybe a little racy due to when the value of outstanding is retrieved. In general though the + * latch should be decremented before the shutdown completion. + * + * @param outstanding + * the number of record processor still awaiting shutdown. + */ + private boolean workerShutdownWithRemaining(long outstanding, GracefulShutdownContext context) { + if (isWorkerShutdownComplete(context)) { + if (outstanding != 0) { + log.info("Shutdown completed, but shutdownCompleteLatch still had outstanding " + outstanding + + " with a current value of " + context.getShutdownCompleteLatch().getCount() + ". shutdownComplete: " + + context.getWorker().isShutdownComplete() + " -- Consumer Map: " + + context.getWorker().getShardInfoShardConsumerMap().size()); + return true; + } + } + return false; + } + + @Override + public Boolean call() throws Exception { + GracefulShutdownContext context; + try { + context = startWorkerShutdown.call(); + } catch (Exception ex) { + log.warn("Caught exception while requesting initial worker shutdown.", ex); + throw ex; + } + return context.isShutdownAlreadyCompleted() || waitForRecordProcessors(context); + } + } +} diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/InitializeTask.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/InitializeTask.java index 262b98c7..5e847a89 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/InitializeTask.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/InitializeTask.java @@ -1,16 +1,16 @@ /* - * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; @@ -19,6 +19,7 @@ import org.apache.commons.logging.LogFactory; import com.amazonaws.services.kinesis.clientlibrary.interfaces.ICheckpoint; import com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessor; +import com.amazonaws.services.kinesis.clientlibrary.lib.checkpoint.Checkpoint; import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber; import com.amazonaws.services.kinesis.clientlibrary.types.InitializationInput; import com.amazonaws.services.kinesis.metrics.impl.MetricsHelper; @@ -42,17 +43,19 @@ class InitializeTask implements ITask { // Back off for this interval if we encounter a problem (exception) private final long backoffTimeMillis; private final StreamConfig streamConfig; + private final GetRecordsCache getRecordsCache; /** * Constructor. */ InitializeTask(ShardInfo shardInfo, - IRecordProcessor recordProcessor, - ICheckpoint checkpoint, - RecordProcessorCheckpointer recordProcessorCheckpointer, - KinesisDataFetcher dataFetcher, - long backoffTimeMillis, - StreamConfig streamConfig) { + IRecordProcessor recordProcessor, + ICheckpoint checkpoint, + RecordProcessorCheckpointer recordProcessorCheckpointer, + KinesisDataFetcher dataFetcher, + long backoffTimeMillis, + StreamConfig streamConfig, + GetRecordsCache getRecordsCache) { this.shardInfo = shardInfo; this.recordProcessor = recordProcessor; this.checkpoint = checkpoint; @@ -60,6 +63,7 @@ class InitializeTask implements ITask { this.dataFetcher = dataFetcher; this.backoffTimeMillis = backoffTimeMillis; this.streamConfig = streamConfig; + this.getRecordsCache = getRecordsCache; } /* @@ -75,16 +79,19 @@ class InitializeTask implements ITask { try { LOG.debug("Initializing ShardId " + shardInfo.getShardId()); - ExtendedSequenceNumber initialCheckpoint = checkpoint.getCheckpoint(shardInfo.getShardId()); + Checkpoint initialCheckpointObject = checkpoint.getCheckpointObject(shardInfo.getShardId()); + ExtendedSequenceNumber initialCheckpoint = initialCheckpointObject.getCheckpoint(); dataFetcher.initialize(initialCheckpoint.getSequenceNumber(), streamConfig.getInitialPositionInStream()); + getRecordsCache.start(); recordProcessorCheckpointer.setLargestPermittedCheckpointValue(initialCheckpoint); recordProcessorCheckpointer.setInitialCheckpointValue(initialCheckpoint); LOG.debug("Calling the record processor initialize()."); final InitializationInput initializationInput = new InitializationInput() .withShardId(shardInfo.getShardId()) - .withExtendedSequenceNumber(initialCheckpoint); + .withExtendedSequenceNumber(initialCheckpoint) + .withPendingCheckpointSequenceNumber(initialCheckpointObject.getPendingCheckpoint()); final long recordProcessorStartTimeMillis = System.currentTimeMillis(); try { recordProcessor.initialize(initializationInput); diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java index b8218968..2e3b4dc3 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java @@ -1,22 +1,25 @@ /* - * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; import java.util.Date; +import java.util.Optional; import java.util.Set; +import org.apache.commons.lang3.Validate; + import com.amazonaws.ClientConfiguration; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.regions.RegionUtils; @@ -25,6 +28,8 @@ import com.amazonaws.services.kinesis.metrics.interfaces.IMetricsScope; import com.amazonaws.services.kinesis.metrics.interfaces.MetricsLevel; import com.google.common.collect.ImmutableSet; +import lombok.Getter; + /** * Configuration for the Amazon Kinesis Client Library. */ @@ -52,7 +57,8 @@ public class KinesisClientLibConfiguration { public static final int DEFAULT_MAX_RECORDS = 10000; /** - * Idle time between record reads in milliseconds. + * The default value for how long the {@link ShardConsumer} should sleep if no records are returned from the call to + * {@link com.amazonaws.services.kinesis.AmazonKinesis#getRecords(com.amazonaws.services.kinesis.model.GetRecordsRequest)}. */ public static final long DEFAULT_IDLETIME_BETWEEN_READS_MILLIS = 1000L; @@ -120,7 +126,7 @@ public class KinesisClientLibConfiguration { /** * User agent set when Amazon Kinesis Client Library makes AWS requests. */ - public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.7.4"; + public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.9.3"; /** * KCL will validate client provided sequence numbers with a call to Amazon Kinesis before checkpointing for calls @@ -154,10 +160,10 @@ public class KinesisClientLibConfiguration { */ public static final int DEFAULT_INITIAL_LEASE_TABLE_WRITE_CAPACITY = 10; - /* - * The Worker will skip shard sync during initialization if there are one or more leases in the lease table. - * This assumes that the shards and leases are in-sync. - * This enables customers to choose faster startup times (e.g. during incremental deployments of an application). + /** + * The Worker will skip shard sync during initialization if there are one or more leases in the lease table. This + * assumes that the shards and leases are in-sync. This enables customers to choose faster startup times (e.g. + * during incremental deployments of an application). */ public static final boolean DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST = false; @@ -166,6 +172,26 @@ public class KinesisClientLibConfiguration { */ public static final ShardPrioritization DEFAULT_SHARD_PRIORITIZATION = new NoOpShardPrioritization(); + /** + * The amount of milliseconds to wait before graceful shutdown forcefully terminates. + */ + public static final long DEFAULT_SHUTDOWN_GRACE_MILLIS = 5000L; + + /** + * The size of the thread pool to create for the lease renewer to use. + */ + public static final int DEFAULT_MAX_LEASE_RENEWAL_THREADS = 20; + + /** + * The sleep time between two listShards calls from the proxy when throttled. + */ + public static final long DEFAULT_LIST_SHARDS_BACKOFF_TIME_IN_MILLIS = 1500; + + /** + * The number of times the Proxy will retry listShards call when throttled. + */ + public static final int DEFAULT_MAX_LIST_SHARDS_RETRY_ATTEMPTS = 50; + private String applicationName; private String tableName; private String streamName; @@ -184,6 +210,7 @@ public class KinesisClientLibConfiguration { private boolean callProcessRecordsEvenForEmptyRecordList; private long parentShardPollIntervalMillis; private boolean cleanupLeasesUponShardCompletion; + private boolean ignoreUnexpectedChildShards; private ClientConfiguration kinesisClientConfig; private ClientConfiguration dynamoDBClientConfig; private ClientConfiguration cloudWatchClientConfig; @@ -202,6 +229,31 @@ public class KinesisClientLibConfiguration { // This is useful for optimizing deployments to large fleets working on a stable stream. private boolean skipShardSyncAtWorkerInitializationIfLeasesExist; private ShardPrioritization shardPrioritization; + private long shutdownGraceMillis; + + @Getter + private Optional timeoutInSeconds = Optional.empty(); + + @Getter + private Optional retryGetRecordsInSeconds = Optional.empty(); + + @Getter + private Optional maxGetRecordsThreadPool = Optional.empty(); + + @Getter + private int maxLeaseRenewalThreads = DEFAULT_MAX_LEASE_RENEWAL_THREADS; + + @Getter + private RecordsFetcherFactory recordsFetcherFactory; + + @Getter + private Optional logWarningForTaskAfterMillis = Optional.empty(); + + @Getter + private long listShardsBackoffTimeInMillis = DEFAULT_LIST_SHARDS_BACKOFF_TIME_IN_MILLIS; + + @Getter + private int maxListShardsRetryAttempts = DEFAULT_MAX_LIST_SHARDS_RETRY_ATTEMPTS; /** * Constructor. @@ -238,14 +290,31 @@ public class KinesisClientLibConfiguration { AWSCredentialsProvider dynamoDBCredentialsProvider, AWSCredentialsProvider cloudWatchCredentialsProvider, String workerId) { - this(applicationName, streamName, null, null, DEFAULT_INITIAL_POSITION_IN_STREAM, kinesisCredentialsProvider, - dynamoDBCredentialsProvider, cloudWatchCredentialsProvider, DEFAULT_FAILOVER_TIME_MILLIS, workerId, - DEFAULT_MAX_RECORDS, DEFAULT_IDLETIME_BETWEEN_READS_MILLIS, - DEFAULT_DONT_CALL_PROCESS_RECORDS_FOR_EMPTY_RECORD_LIST, DEFAULT_PARENT_SHARD_POLL_INTERVAL_MILLIS, - DEFAULT_SHARD_SYNC_INTERVAL_MILLIS, DEFAULT_CLEANUP_LEASES_UPON_SHARDS_COMPLETION, - new ClientConfiguration(), new ClientConfiguration(), new ClientConfiguration(), - DEFAULT_TASK_BACKOFF_TIME_MILLIS, DEFAULT_METRICS_BUFFER_TIME_MILLIS, DEFAULT_METRICS_MAX_QUEUE_SIZE, - DEFAULT_VALIDATE_SEQUENCE_NUMBER_BEFORE_CHECKPOINTING, null); + this(applicationName, + streamName, + null, + null, + DEFAULT_INITIAL_POSITION_IN_STREAM, + kinesisCredentialsProvider, + dynamoDBCredentialsProvider, + cloudWatchCredentialsProvider, + DEFAULT_FAILOVER_TIME_MILLIS, + workerId, + DEFAULT_MAX_RECORDS, + DEFAULT_IDLETIME_BETWEEN_READS_MILLIS, + DEFAULT_DONT_CALL_PROCESS_RECORDS_FOR_EMPTY_RECORD_LIST, + DEFAULT_PARENT_SHARD_POLL_INTERVAL_MILLIS, + DEFAULT_SHARD_SYNC_INTERVAL_MILLIS, + DEFAULT_CLEANUP_LEASES_UPON_SHARDS_COMPLETION, + new ClientConfiguration(), + new ClientConfiguration(), + new ClientConfiguration(), + DEFAULT_TASK_BACKOFF_TIME_MILLIS, + DEFAULT_METRICS_BUFFER_TIME_MILLIS, + DEFAULT_METRICS_MAX_QUEUE_SIZE, + DEFAULT_VALIDATE_SEQUENCE_NUMBER_BEFORE_CHECKPOINTING, + null, + DEFAULT_SHUTDOWN_GRACE_MILLIS); } /** @@ -280,32 +349,34 @@ public class KinesisClientLibConfiguration { * with a call to Amazon Kinesis before checkpointing for calls to * {@link RecordProcessorCheckpointer#checkpoint(String)} * @param regionName The region name for the service + * @param shutdownGraceMillis The number of milliseconds before graceful shutdown terminates forcefully */ // CHECKSTYLE:IGNORE HiddenFieldCheck FOR NEXT 26 LINES // CHECKSTYLE:IGNORE ParameterNumber FOR NEXT 26 LINES public KinesisClientLibConfiguration(String applicationName, - String streamName, - String kinesisEndpoint, - InitialPositionInStream initialPositionInStream, - AWSCredentialsProvider kinesisCredentialsProvider, - AWSCredentialsProvider dynamoDBCredentialsProvider, - AWSCredentialsProvider cloudWatchCredentialsProvider, - long failoverTimeMillis, - String workerId, - int maxRecords, - long idleTimeBetweenReadsInMillis, - boolean callProcessRecordsEvenForEmptyRecordList, - long parentShardPollIntervalMillis, - long shardSyncIntervalMillis, - boolean cleanupTerminatedShardsBeforeExpiry, - ClientConfiguration kinesisClientConfig, - ClientConfiguration dynamoDBClientConfig, - ClientConfiguration cloudWatchClientConfig, - long taskBackoffTimeMillis, - long metricsBufferTimeMillis, - int metricsMaxQueueSize, - boolean validateSequenceNumberBeforeCheckpointing, - String regionName) { + String streamName, + String kinesisEndpoint, + InitialPositionInStream initialPositionInStream, + AWSCredentialsProvider kinesisCredentialsProvider, + AWSCredentialsProvider dynamoDBCredentialsProvider, + AWSCredentialsProvider cloudWatchCredentialsProvider, + long failoverTimeMillis, + String workerId, + int maxRecords, + long idleTimeBetweenReadsInMillis, + boolean callProcessRecordsEvenForEmptyRecordList, + long parentShardPollIntervalMillis, + long shardSyncIntervalMillis, + boolean cleanupTerminatedShardsBeforeExpiry, + ClientConfiguration kinesisClientConfig, + ClientConfiguration dynamoDBClientConfig, + ClientConfiguration cloudWatchClientConfig, + long taskBackoffTimeMillis, + long metricsBufferTimeMillis, + int metricsMaxQueueSize, + boolean validateSequenceNumberBeforeCheckpointing, + String regionName, + long shutdownGraceMillis) { this(applicationName, streamName, kinesisEndpoint, null, initialPositionInStream, kinesisCredentialsProvider, dynamoDBCredentialsProvider, cloudWatchCredentialsProvider, failoverTimeMillis, workerId, maxRecords, idleTimeBetweenReadsInMillis, @@ -313,7 +384,116 @@ public class KinesisClientLibConfiguration { shardSyncIntervalMillis, cleanupTerminatedShardsBeforeExpiry, kinesisClientConfig, dynamoDBClientConfig, cloudWatchClientConfig, taskBackoffTimeMillis, metricsBufferTimeMillis, metricsMaxQueueSize, - validateSequenceNumberBeforeCheckpointing, regionName); + validateSequenceNumberBeforeCheckpointing, regionName, shutdownGraceMillis); + } + + /** + * @param applicationName Name of the Kinesis application + * By default the application name is included in the user agent string used to make AWS requests. This + * can assist with troubleshooting (e.g. distinguish requests made by separate applications). + * @param streamName Name of the Kinesis stream + * @param kinesisEndpoint Kinesis endpoint + * @param dynamoDBEndpoint DynamoDB endpoint + * @param initialPositionInStream One of LATEST or TRIM_HORIZON. The KinesisClientLibrary will start fetching + * records from that location in the stream when an application starts up for the first time and there + * are no checkpoints. If there are checkpoints, then we start from the checkpoint position. + * @param kinesisCredentialsProvider Provides credentials used to access Kinesis + * @param dynamoDBCredentialsProvider Provides credentials used to access DynamoDB + * @param cloudWatchCredentialsProvider Provides credentials used to access CloudWatch + * @param failoverTimeMillis Lease duration (leases not renewed within this period will be claimed by others) + * @param workerId Used to distinguish different workers/processes of a Kinesis application + * @param maxRecords Max records to read per Kinesis getRecords() call + * @param idleTimeBetweenReadsInMillis Idle time between calls to fetch data from Kinesis + * @param callProcessRecordsEvenForEmptyRecordList Call the IRecordProcessor::processRecords() API even if + * GetRecords returned an empty record list. + * @param parentShardPollIntervalMillis Wait for this long between polls to check if parent shards are done + * @param shardSyncIntervalMillis Time between tasks to sync leases and Kinesis shards + * @param cleanupTerminatedShardsBeforeExpiry Clean up shards we've finished processing (don't wait for expiration + * in Kinesis) + * @param kinesisClientConfig Client Configuration used by Kinesis client + * @param dynamoDBClientConfig Client Configuration used by DynamoDB client + * @param cloudWatchClientConfig Client Configuration used by CloudWatch client + * @param taskBackoffTimeMillis Backoff period when tasks encounter an exception + * @param metricsBufferTimeMillis Metrics are buffered for at most this long before publishing to CloudWatch + * @param metricsMaxQueueSize Max number of metrics to buffer before publishing to CloudWatch + * @param validateSequenceNumberBeforeCheckpointing whether KCL should validate client provided sequence numbers + * with a call to Amazon Kinesis before checkpointing for calls to + * {@link RecordProcessorCheckpointer#checkpoint(String)} + * @param regionName The region name for the service + */ + // CHECKSTYLE:IGNORE HiddenFieldCheck FOR NEXT 26 LINES + // CHECKSTYLE:IGNORE ParameterNumber FOR NEXT 26 LINES + public KinesisClientLibConfiguration(String applicationName, + String streamName, + String kinesisEndpoint, + String dynamoDBEndpoint, + InitialPositionInStream initialPositionInStream, + AWSCredentialsProvider kinesisCredentialsProvider, + AWSCredentialsProvider dynamoDBCredentialsProvider, + AWSCredentialsProvider cloudWatchCredentialsProvider, + long failoverTimeMillis, + String workerId, + int maxRecords, + long idleTimeBetweenReadsInMillis, + boolean callProcessRecordsEvenForEmptyRecordList, + long parentShardPollIntervalMillis, + long shardSyncIntervalMillis, + boolean cleanupTerminatedShardsBeforeExpiry, + ClientConfiguration kinesisClientConfig, + ClientConfiguration dynamoDBClientConfig, + ClientConfiguration cloudWatchClientConfig, + long taskBackoffTimeMillis, + long metricsBufferTimeMillis, + int metricsMaxQueueSize, + boolean validateSequenceNumberBeforeCheckpointing, + String regionName, + long shutdownGraceMillis) { + // Check following values are greater than zero + checkIsValuePositive("FailoverTimeMillis", failoverTimeMillis); + checkIsValuePositive("IdleTimeBetweenReadsInMillis", idleTimeBetweenReadsInMillis); + checkIsValuePositive("ParentShardPollIntervalMillis", parentShardPollIntervalMillis); + checkIsValuePositive("ShardSyncIntervalMillis", shardSyncIntervalMillis); + checkIsValuePositive("MaxRecords", (long) maxRecords); + checkIsValuePositive("TaskBackoffTimeMillis", taskBackoffTimeMillis); + checkIsValuePositive("MetricsBufferTimeMills", metricsBufferTimeMillis); + checkIsValuePositive("MetricsMaxQueueSize", (long) metricsMaxQueueSize); + checkIsValuePositive("ShutdownGraceMillis", shutdownGraceMillis); + this.applicationName = applicationName; + this.tableName = applicationName; + this.streamName = streamName; + this.kinesisEndpoint = kinesisEndpoint; + this.dynamoDBEndpoint = dynamoDBEndpoint; + this.initialPositionInStream = initialPositionInStream; + this.kinesisCredentialsProvider = kinesisCredentialsProvider; + this.dynamoDBCredentialsProvider = dynamoDBCredentialsProvider; + this.cloudWatchCredentialsProvider = cloudWatchCredentialsProvider; + this.failoverTimeMillis = failoverTimeMillis; + this.maxRecords = maxRecords; + this.idleTimeBetweenReadsInMillis = idleTimeBetweenReadsInMillis; + this.callProcessRecordsEvenForEmptyRecordList = callProcessRecordsEvenForEmptyRecordList; + this.parentShardPollIntervalMillis = parentShardPollIntervalMillis; + this.shardSyncIntervalMillis = shardSyncIntervalMillis; + this.cleanupLeasesUponShardCompletion = cleanupTerminatedShardsBeforeExpiry; + this.workerIdentifier = workerId; + this.kinesisClientConfig = checkAndAppendKinesisClientLibUserAgent(kinesisClientConfig); + this.dynamoDBClientConfig = checkAndAppendKinesisClientLibUserAgent(dynamoDBClientConfig); + this.cloudWatchClientConfig = checkAndAppendKinesisClientLibUserAgent(cloudWatchClientConfig); + this.taskBackoffTimeMillis = taskBackoffTimeMillis; + this.metricsBufferTimeMillis = metricsBufferTimeMillis; + this.metricsMaxQueueSize = metricsMaxQueueSize; + this.metricsLevel = DEFAULT_METRICS_LEVEL; + this.metricsEnabledDimensions = DEFAULT_METRICS_ENABLED_DIMENSIONS; + this.validateSequenceNumberBeforeCheckpointing = validateSequenceNumberBeforeCheckpointing; + this.regionName = regionName; + this.maxLeasesForWorker = DEFAULT_MAX_LEASES_FOR_WORKER; + this.maxLeasesToStealAtOneTime = DEFAULT_MAX_LEASES_TO_STEAL_AT_ONE_TIME; + this.initialLeaseTableReadCapacity = DEFAULT_INITIAL_LEASE_TABLE_READ_CAPACITY; + this.initialLeaseTableWriteCapacity = DEFAULT_INITIAL_LEASE_TABLE_WRITE_CAPACITY; + this.initialPositionInStreamExtended = + InitialPositionInStreamExtended.newInitialPosition(initialPositionInStream); + this.skipShardSyncAtWorkerInitializationIfLeasesExist = DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST; + this.shardPrioritization = DEFAULT_SHARD_PRIORITIZATION; + this.recordsFetcherFactory = new SimpleRecordsFetcherFactory(); } /** @@ -375,7 +555,8 @@ public class KinesisClientLibConfiguration { long metricsBufferTimeMillis, int metricsMaxQueueSize, boolean validateSequenceNumberBeforeCheckpointing, - String regionName) { + String regionName, + RecordsFetcherFactory recordsFetcherFactory) { // Check following values are greater than zero checkIsValuePositive("FailoverTimeMillis", failoverTimeMillis); checkIsValuePositive("IdleTimeBetweenReadsInMillis", idleTimeBetweenReadsInMillis); @@ -385,7 +566,6 @@ public class KinesisClientLibConfiguration { checkIsValuePositive("TaskBackoffTimeMillis", taskBackoffTimeMillis); checkIsValuePositive("MetricsBufferTimeMills", metricsBufferTimeMillis); checkIsValuePositive("MetricsMaxQueueSize", (long) metricsMaxQueueSize); - checkIsRegionNameValid(regionName); this.applicationName = applicationName; this.tableName = applicationName; this.streamName = streamName; @@ -421,6 +601,8 @@ public class KinesisClientLibConfiguration { InitialPositionInStreamExtended.newInitialPosition(initialPositionInStream); this.skipShardSyncAtWorkerInitializationIfLeasesExist = DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST; this.shardPrioritization = DEFAULT_SHARD_PRIORITIZATION; + this.recordsFetcherFactory = recordsFetcherFactory; + this.shutdownGraceMillis = shutdownGraceMillis; } // Check if value is positive, otherwise throw an exception @@ -446,12 +628,6 @@ public class KinesisClientLibConfiguration { return config; } - private void checkIsRegionNameValid(String regionNameToCheck) { - if (regionNameToCheck != null && RegionUtils.getRegion(regionNameToCheck) == null) { - throw new IllegalArgumentException("The specified region name is not valid"); - } - } - /** * @return Name of the application */ @@ -635,6 +811,13 @@ public class KinesisClientLibConfiguration { return cleanupLeasesUponShardCompletion; } + /** + * @return true if we should ignore child shards which have open parents + */ + public boolean shouldIgnoreUnexpectedChildShards() { + return ignoreUnexpectedChildShards; + } + /** * @return true if KCL should validate client provided sequence numbers with a call to Amazon Kinesis before * checkpointing for calls to {@link RecordProcessorCheckpointer#checkpoint(String)} @@ -708,6 +891,14 @@ public class KinesisClientLibConfiguration { return shardPrioritization; } + /** + * @return Graceful shutdown timeout + */ + public long getShutdownGraceMillis() { + return shutdownGraceMillis; + } + + /* // CHECKSTYLE:IGNORE HiddenFieldCheck FOR NEXT 190 LINES /** * @param tableName name of the lease table in DynamoDB @@ -790,7 +981,23 @@ public class KinesisClientLibConfiguration { } /** - * @param idleTimeBetweenReadsInMillis Idle time between calls to fetch data from Kinesis + * Controls how long the KCL will sleep if no records are returned from Kinesis + * + *

+ * This value is only used when no records are returned; if records are returned, the {@link com.amazonaws.services.kinesis.clientlibrary.lib.worker.ProcessTask} will + * immediately retrieve the next set of records after the call to + * {@link com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessor#processRecords(ProcessRecordsInput)} + * has returned. Setting this value to high may result in the KCL being unable to catch up. If you are changing this + * value it's recommended that you enable {@link #withCallProcessRecordsEvenForEmptyRecordList(boolean)}, and + * monitor how far behind the records retrieved are by inspecting + * {@link com.amazonaws.services.kinesis.clientlibrary.types.ProcessRecordsInput#getMillisBehindLatest()}, and the + * CloudWatch + * Metric: GetRecords.MillisBehindLatest + *

+ * + * @param idleTimeBetweenReadsInMillis + * how long to sleep between GetRecords calls when no records are returned. * @return KinesisClientLibConfiguration */ public KinesisClientLibConfiguration withIdleTimeBetweenReadsInMillis(long idleTimeBetweenReadsInMillis) { @@ -831,6 +1038,16 @@ public class KinesisClientLibConfiguration { return this; } + /** + * @param ignoreUnexpectedChildShards Ignore child shards with open parents. + * @return KinesisClientLibConfiguration + */ + public KinesisClientLibConfiguration withIgnoreUnexpectedChildShards( + boolean ignoreUnexpectedChildShards) { + this.ignoreUnexpectedChildShards = ignoreUnexpectedChildShards; + return this; + } + /** * @param clientConfig Common client configuration used by Kinesis/DynamoDB/CloudWatch client * @return KinesisClientLibConfiguration @@ -989,7 +1206,6 @@ public class KinesisClientLibConfiguration { */ // CHECKSTYLE:IGNORE HiddenFieldCheck FOR NEXT 2 LINES public KinesisClientLibConfiguration withRegionName(String regionName) { - checkIsRegionNameValid(regionName); this.regionName = regionName; return this; } @@ -1058,4 +1274,146 @@ public class KinesisClientLibConfiguration { this.shardPrioritization = shardPrioritization; return this; } + + /** + * Sets the size of the thread pool that will be used to renew leases. + * + * Setting this to low may starve the lease renewal process, and cause the worker to lose leases at a higher rate. + * + * @param maxLeaseRenewalThreads + * the maximum size of the lease renewal thread pool + * @throws IllegalArgumentException + * if maxLeaseRenewalThreads is <= 0 + * @return this configuration object + */ + public KinesisClientLibConfiguration withMaxLeaseRenewalThreads(int maxLeaseRenewalThreads) { + Validate.isTrue(maxLeaseRenewalThreads > 2, + "The maximum number of lease renewal threads must be greater than or equal to 2."); + this.maxLeaseRenewalThreads = maxLeaseRenewalThreads; + + return this; + } + + + /** + * @param retryGetRecordsInSeconds the time in seconds to wait before the worker retries to get a record. + * @return this configuration object. + */ + public KinesisClientLibConfiguration withRetryGetRecordsInSeconds(final int retryGetRecordsInSeconds) { + checkIsValuePositive("retryGetRecordsInSeconds", retryGetRecordsInSeconds); + this.retryGetRecordsInSeconds = Optional.of(retryGetRecordsInSeconds); + return this; + } + + /** + *@param maxGetRecordsThreadPool the max number of threads in the getRecords thread pool. + *@return this configuration object + */ + public KinesisClientLibConfiguration withMaxGetRecordsThreadPool(final int maxGetRecordsThreadPool) { + checkIsValuePositive("maxGetRecordsThreadPool", maxGetRecordsThreadPool); + this.maxGetRecordsThreadPool = Optional.of(maxGetRecordsThreadPool); + return this; + } + + /** + * + * @param maxPendingProcessRecordsInput The max number of ProcessRecordsInput that can be stored in the cache before + * blocking + * @return this configuration object + */ + public KinesisClientLibConfiguration withMaxPendingProcessRecordsInput(final int maxPendingProcessRecordsInput) { + checkIsValuePositive("maxPendingProcessRecordsInput", maxPendingProcessRecordsInput); + this.recordsFetcherFactory.setMaxPendingProcessRecordsInput(maxPendingProcessRecordsInput); + return this; + } + + /** + * @param maxCacheByteSize Max byte size for the cache at any given point of time. After this threshold is crossed + * the KinesisDataFetcher will be blocked until the cache has more space available. + * @return KinesisClientLibConfiguration + */ + public KinesisClientLibConfiguration withMaxCacheByteSize(final int maxCacheByteSize) { + checkIsValuePositive("maxCacheByteSize", maxCacheByteSize); + this.recordsFetcherFactory.setMaxByteSize(maxCacheByteSize); + return this; + } + + /** + * @param dataFetchingStrategy The strategy for fetching data from kinesis. + * @return KinesisClientLibConfiguration + */ + public KinesisClientLibConfiguration withDataFetchingStrategy(String dataFetchingStrategy) { + this.recordsFetcherFactory.setDataFetchingStrategy(DataFetchingStrategy.valueOf(dataFetchingStrategy.toUpperCase())); + return this; + } + + /** + * @param maxRecordsCount The maximum number of records in the cache, accross all ProcessRecordInput objects + * @return KinesisClientLibConfiguration + */ + public KinesisClientLibConfiguration withMaxRecordsCount(final int maxRecordsCount) { + checkIsValuePositive("maxRecordsCount", maxRecordsCount); + this.recordsFetcherFactory.setMaxRecordsCount(maxRecordsCount); + return this; + } + + /** + * @param timeoutInSeconds The timeout in seconds to wait for the MultiLangProtocol to wait for + */ + public void withTimeoutInSeconds(final int timeoutInSeconds) { + this.timeoutInSeconds = Optional.of(timeoutInSeconds); + } + + /** + * @param shutdownGraceMillis Time before gracefully shutdown forcefully terminates + * @return KinesisClientLibConfiguration + */ + public KinesisClientLibConfiguration withShutdownGraceMillis(long shutdownGraceMillis) { + checkIsValuePositive("ShutdownGraceMillis", shutdownGraceMillis); + this.shutdownGraceMillis = shutdownGraceMillis; + return this; + } + + /** + * @param idleMillisBetweenCalls Idle time between 2 getcalls from the data fetcher. + * @return KinesisClientLibConfiguration + */ + public KinesisClientLibConfiguration withIdleMillisBetweenCalls(long idleMillisBetweenCalls) { + checkIsValuePositive("IdleMillisBetweenCalls", idleMillisBetweenCalls); + this.recordsFetcherFactory.setIdleMillisBetweenCalls(idleMillisBetweenCalls); + return this; + } + + /** + * @param logWarningForTaskAfterMillis Logs warn message if as task is held in a task for more than the set + * time. + * @return KinesisClientLibConfiguration + */ + public KinesisClientLibConfiguration withLogWarningForTaskAfterMillis(long logWarningForTaskAfterMillis) { + checkIsValuePositive("LogProcessTaskStatusAfterInMillis", logWarningForTaskAfterMillis); + this.logWarningForTaskAfterMillis = Optional.of(logWarningForTaskAfterMillis); + return this; + } + + /** + * @param listShardsBackoffTimeInMillis Max sleep between two listShards call when throttled + * in {@link com.amazonaws.services.kinesis.clientlibrary.proxies.KinesisProxy}. + * @return + */ + public KinesisClientLibConfiguration withListShardsBackoffTimeInMillis(long listShardsBackoffTimeInMillis) { + checkIsValuePositive("listShardsBackoffTimeInMillis", listShardsBackoffTimeInMillis); + this.listShardsBackoffTimeInMillis = listShardsBackoffTimeInMillis; + return this; + } + + /** + * @param maxListShardsRetryAttempts Max number of retries for listShards when throttled + * in {@link com.amazonaws.services.kinesis.clientlibrary.proxies.KinesisProxy}. + * @return + */ + public KinesisClientLibConfiguration withMaxListShardsRetryAttempts(int maxListShardsRetryAttempts) { + checkIsValuePositive("maxListShardsRetryAttempts", maxListShardsRetryAttempts); + this.maxListShardsRetryAttempts = maxListShardsRetryAttempts; + return this; + } } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibLeaseCoordinator.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibLeaseCoordinator.java index 59de31be..448a2953 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibLeaseCoordinator.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibLeaseCoordinator.java @@ -18,6 +18,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.UUID; @@ -30,6 +31,7 @@ import com.amazonaws.services.kinesis.clientlibrary.exceptions.ShutdownException import com.amazonaws.services.kinesis.clientlibrary.exceptions.ThrottlingException; import com.amazonaws.services.kinesis.clientlibrary.exceptions.internal.KinesisClientLibIOException; import com.amazonaws.services.kinesis.clientlibrary.interfaces.ICheckpoint; +import com.amazonaws.services.kinesis.clientlibrary.lib.checkpoint.Checkpoint; import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber; import com.amazonaws.services.kinesis.leases.exceptions.DependencyException; import com.amazonaws.services.kinesis.leases.exceptions.InvalidStateException; @@ -99,9 +101,10 @@ class KinesisClientLibLeaseCoordinator extends LeaseCoordinator getRecordsResultQueue; + private int maxPendingProcessRecordsInput; + private int maxByteSize; + private int maxRecordsCount; + private final int maxRecordsPerCall; + private final GetRecordsRetrievalStrategy getRecordsRetrievalStrategy; + private final ExecutorService executorService; + private final IMetricsFactory metricsFactory; + private final long idleMillisBetweenCalls; + private Instant lastSuccessfulCall; + private final DefaultGetRecordsCacheDaemon defaultGetRecordsCacheDaemon; + private PrefetchCounters prefetchCounters; + private boolean started = false; + private final String operation; + private final KinesisDataFetcher dataFetcher; + private final String shardId; + + /** + * Constructor for the PrefetchGetRecordsCache. This cache prefetches records from Kinesis and stores them in a + * LinkedBlockingQueue. + * + * @see com.amazonaws.services.kinesis.clientlibrary.lib.worker.PrefetchGetRecordsCache + * + * @param maxPendingProcessRecordsInput Max number of ProcessRecordsInput that can be held in the cache before + * blocking + * @param maxByteSize Max byte size of the queue before blocking next get records call + * @param maxRecordsCount Max number of records in the queue across all ProcessRecordInput objects + * @param maxRecordsPerCall Max records to be returned per call + * @param getRecordsRetrievalStrategy Retrieval strategy for the get records call + * @param executorService Executor service for the cache + * @param idleMillisBetweenCalls maximum time to wait before dispatching the next get records call + */ + public PrefetchGetRecordsCache(final int maxPendingProcessRecordsInput, final int maxByteSize, final int maxRecordsCount, + final int maxRecordsPerCall, + @NonNull final GetRecordsRetrievalStrategy getRecordsRetrievalStrategy, + @NonNull final ExecutorService executorService, + final long idleMillisBetweenCalls, + @NonNull final IMetricsFactory metricsFactory, + @NonNull final String operation, + @NonNull final String shardId) { + this.getRecordsRetrievalStrategy = getRecordsRetrievalStrategy; + this.maxRecordsPerCall = maxRecordsPerCall; + this.maxPendingProcessRecordsInput = maxPendingProcessRecordsInput; + this.maxByteSize = maxByteSize; + this.maxRecordsCount = maxRecordsCount; + this.getRecordsResultQueue = new LinkedBlockingQueue<>(this.maxPendingProcessRecordsInput); + this.prefetchCounters = new PrefetchCounters(); + this.executorService = executorService; + this.metricsFactory = new ThreadSafeMetricsDelegatingFactory(metricsFactory); + this.idleMillisBetweenCalls = idleMillisBetweenCalls; + this.defaultGetRecordsCacheDaemon = new DefaultGetRecordsCacheDaemon(); + Validate.notEmpty(operation, "Operation cannot be empty"); + this.operation = operation; + this.dataFetcher = this.getRecordsRetrievalStrategy.getDataFetcher(); + this.shardId = shardId; + } + + @Override + public void start() { + if (executorService.isShutdown()) { + throw new IllegalStateException("ExecutorService has been shutdown."); + } + + if (!started) { + log.info("Starting prefetching thread."); + executorService.execute(defaultGetRecordsCacheDaemon); + } + started = true; + } + + @Override + public ProcessRecordsInput getNextResult() { + if (executorService.isShutdown()) { + throw new IllegalStateException("Shutdown has been called on the cache, can't accept new requests."); + } + + if (!started) { + throw new IllegalStateException("Cache has not been initialized, make sure to call start."); + } + ProcessRecordsInput result = null; + try { + result = getRecordsResultQueue.take().withCacheExitTime(Instant.now()); + prefetchCounters.removed(result); + } catch (InterruptedException e) { + log.error("Interrupted while getting records from the cache", e); + } + return result; + } + + @Override + public GetRecordsRetrievalStrategy getGetRecordsRetrievalStrategy() { + return getRecordsRetrievalStrategy; + } + + @Override + public void shutdown() { + defaultGetRecordsCacheDaemon.isShutdown = true; + executorService.shutdownNow(); + started = false; + } + + private class DefaultGetRecordsCacheDaemon implements Runnable { + volatile boolean isShutdown = false; + + @Override + public void run() { + while (!isShutdown) { + if (Thread.currentThread().isInterrupted()) { + log.warn("Prefetch thread was interrupted."); + break; + } + MetricsHelper.startScope(metricsFactory, operation); + if (prefetchCounters.shouldGetNewRecords()) { + try { + sleepBeforeNextCall(); + GetRecordsResult getRecordsResult = getRecordsRetrievalStrategy.getRecords(maxRecordsPerCall); + lastSuccessfulCall = Instant.now(); + ProcessRecordsInput processRecordsInput = new ProcessRecordsInput() + .withRecords(getRecordsResult.getRecords()) + .withMillisBehindLatest(getRecordsResult.getMillisBehindLatest()) + .withCacheEntryTime(lastSuccessfulCall); + getRecordsResultQueue.put(processRecordsInput); + prefetchCounters.added(processRecordsInput); + } catch (InterruptedException e) { + log.info("Thread was interrupted, indicating shutdown was called on the cache."); + } catch (ExpiredIteratorException e) { + log.info(String.format("ShardId %s: getRecords threw ExpiredIteratorException - restarting" + + " after greatest seqNum passed to customer", shardId), e); + + MetricsHelper.getMetricsScope().addData(EXPIRED_ITERATOR_METRIC, 1, StandardUnit.Count, + MetricsLevel.SUMMARY); + + dataFetcher.restartIterator(); + } catch (SdkClientException e) { + log.error("Exception thrown while fetching records from Kinesis", e); + } catch (Throwable e) { + log.error("Unexpected exception was thrown. This could probably be an issue or a bug." + + " Please search for the exception/error online to check what is going on. If the " + + "issue persists or is a recurring problem, feel free to open an issue on, " + + "https://github.com/awslabs/amazon-kinesis-client.", e); + } finally { + MetricsHelper.endScope(); + } + } else { + // + // Consumer isn't ready to receive new records will allow prefetch counters to pause + // + try { + prefetchCounters.waitForConsumer(); + } catch (InterruptedException ie) { + log.info("Thread was interrupted while waiting for the consumer. " + + "Shutdown has probably been started"); + } + } + } + callShutdownOnStrategy(); + } + + private void callShutdownOnStrategy() { + if (!getRecordsRetrievalStrategy.isShutdown()) { + getRecordsRetrievalStrategy.shutdown(); + } + } + + private void sleepBeforeNextCall() throws InterruptedException { + if (lastSuccessfulCall == null) { + return; + } + long timeSinceLastCall = Duration.between(lastSuccessfulCall, Instant.now()).abs().toMillis(); + if (timeSinceLastCall < idleMillisBetweenCalls) { + Thread.sleep(idleMillisBetweenCalls - timeSinceLastCall); + } + } + } + + private class PrefetchCounters { + private long size = 0; + private long byteSize = 0; + + public synchronized void added(final ProcessRecordsInput result) { + size += getSize(result); + byteSize += getByteSize(result); + } + + public synchronized void removed(final ProcessRecordsInput result) { + size -= getSize(result); + byteSize -= getByteSize(result); + this.notifyAll(); + } + + private long getSize(final ProcessRecordsInput result) { + return result.getRecords().size(); + } + + private long getByteSize(final ProcessRecordsInput result) { + return result.getRecords().stream().mapToLong(record -> record.getData().array().length).sum(); + } + + public synchronized void waitForConsumer() throws InterruptedException { + if (!shouldGetNewRecords()) { + log.debug("Queue is full waiting for consumer for " + idleMillisBetweenCalls + " ms"); + this.wait(idleMillisBetweenCalls); + } + } + + public synchronized boolean shouldGetNewRecords() { + if (log.isDebugEnabled()) { + log.debug("Current Prefetch Counter States: " + this.toString()); + } + return size < maxRecordsCount && byteSize < maxByteSize; + } + + @Override + public String toString() { + return String.format("{ Requests: %d, Records: %d, Bytes: %d }", getRecordsResultQueue.size(), size, + byteSize); + } + } + +} diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PreparedCheckpointer.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PreparedCheckpointer.java new file mode 100644 index 00000000..b7b4ba9d --- /dev/null +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PreparedCheckpointer.java @@ -0,0 +1,65 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package com.amazonaws.services.kinesis.clientlibrary.lib.worker; + +import com.amazonaws.services.kinesis.clientlibrary.exceptions.InvalidStateException; +import com.amazonaws.services.kinesis.clientlibrary.exceptions.KinesisClientLibDependencyException; +import com.amazonaws.services.kinesis.clientlibrary.exceptions.ShutdownException; +import com.amazonaws.services.kinesis.clientlibrary.exceptions.ThrottlingException; +import com.amazonaws.services.kinesis.clientlibrary.interfaces.IPreparedCheckpointer; +import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer; +import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber; + +/** + * Objects of this class are prepared to checkpoint at a specific sequence number. They use an + * IRecordProcessorCheckpointer to do the actual checkpointing, so their checkpoint is subject to the same 'didn't go + * backwards' validation as a normal checkpoint. + */ +public class PreparedCheckpointer implements IPreparedCheckpointer { + + private final ExtendedSequenceNumber pendingCheckpointSequenceNumber; + private final IRecordProcessorCheckpointer checkpointer; + + /** + * Constructor. + * + * @param pendingCheckpointSequenceNumber sequence number to checkpoint at + * @param checkpointer checkpointer to use + */ + public PreparedCheckpointer(ExtendedSequenceNumber pendingCheckpointSequenceNumber, + IRecordProcessorCheckpointer checkpointer) { + this.pendingCheckpointSequenceNumber = pendingCheckpointSequenceNumber; + this.checkpointer = checkpointer; + } + + /** + * {@inheritDoc} + */ + @Override + public ExtendedSequenceNumber getPendingCheckpoint() { + return pendingCheckpointSequenceNumber; + } + + /** + * {@inheritDoc} + */ + @Override + public void checkpoint() + throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException, + IllegalArgumentException { + checkpointer.checkpoint(pendingCheckpointSequenceNumber.getSequenceNumber(), + pendingCheckpointSequenceNumber.getSubSequenceNumber()); + } +} diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ProcessTask.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ProcessTask.java index c419c693..9aca832e 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ProcessTask.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ProcessTask.java @@ -1,16 +1,16 @@ /* - * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; @@ -62,6 +62,8 @@ class ProcessTask implements ITask { private final Shard shard; private final ThrottlingReporter throttlingReporter; + private final GetRecordsCache getRecordsCache; + /** * @param shardInfo * contains information about the shard @@ -75,13 +77,17 @@ class ProcessTask implements ITask { * Kinesis data fetcher (used to fetch records from Kinesis) * @param backoffTimeMillis * backoff time when catching exceptions + * @param getRecordsCache + * The retrieval strategy for fetching records from kinesis */ public ProcessTask(ShardInfo shardInfo, StreamConfig streamConfig, IRecordProcessor recordProcessor, - RecordProcessorCheckpointer recordProcessorCheckpointer, KinesisDataFetcher dataFetcher, - long backoffTimeMillis, boolean skipShardSyncAtWorkerInitializationIfLeasesExist) { + RecordProcessorCheckpointer recordProcessorCheckpointer, KinesisDataFetcher dataFetcher, + long backoffTimeMillis, boolean skipShardSyncAtWorkerInitializationIfLeasesExist, + GetRecordsCache getRecordsCache) { this(shardInfo, streamConfig, recordProcessor, recordProcessorCheckpointer, dataFetcher, backoffTimeMillis, skipShardSyncAtWorkerInitializationIfLeasesExist, - new ThrottlingReporter(MAX_CONSECUTIVE_THROTTLES, shardInfo.getShardId())); + new ThrottlingReporter(MAX_CONSECUTIVE_THROTTLES, shardInfo.getShardId()), + getRecordsCache); } /** @@ -101,9 +107,9 @@ class ProcessTask implements ITask { * determines how throttling events should be reported in the log. */ public ProcessTask(ShardInfo shardInfo, StreamConfig streamConfig, IRecordProcessor recordProcessor, - RecordProcessorCheckpointer recordProcessorCheckpointer, KinesisDataFetcher dataFetcher, - long backoffTimeMillis, boolean skipShardSyncAtWorkerInitializationIfLeasesExist, - ThrottlingReporter throttlingReporter) { + RecordProcessorCheckpointer recordProcessorCheckpointer, KinesisDataFetcher dataFetcher, + long backoffTimeMillis, boolean skipShardSyncAtWorkerInitializationIfLeasesExist, + ThrottlingReporter throttlingReporter, GetRecordsCache getRecordsCache) { super(); this.shardInfo = shardInfo; this.recordProcessor = recordProcessor; @@ -113,6 +119,7 @@ class ProcessTask implements ITask { this.backoffTimeMillis = backoffTimeMillis; this.throttlingReporter = throttlingReporter; IKinesisProxy kinesisProxy = this.streamConfig.getStreamProxy(); + this.getRecordsCache = getRecordsCache; // If skipShardSyncAtWorkerInitializationIfLeasesExist is set, we will not get the shard for // this ProcessTask. In this case, duplicate KPL user records in the event of resharding will // not be dropped during deaggregation of Amazon Kinesis records. This is only applicable if @@ -141,7 +148,6 @@ class ProcessTask implements ITask { scope.addDimension(MetricsHelper.SHARD_ID_DIMENSION_NAME, shardInfo.getShardId()); scope.addData(RECORDS_PROCESSED_METRIC, 0, StandardUnit.Count, MetricsLevel.SUMMARY); scope.addData(DATA_BYTES_PROCESSED_METRIC, 0, StandardUnit.Bytes, MetricsLevel.SUMMARY); - Exception exception = null; try { @@ -150,10 +156,10 @@ class ProcessTask implements ITask { return new TaskResult(null, true); } - final GetRecordsResult getRecordsResult = getRecordsResult(); + final ProcessRecordsInput processRecordsInput = getRecordsResult(); throttlingReporter.success(); - List records = getRecordsResult.getRecords(); - + List records = processRecordsInput.getRecords(); + if (!records.isEmpty()) { scope.addData(RECORDS_PROCESSED_METRIC, records.size(), StandardUnit.Count, MetricsLevel.SUMMARY); } else { @@ -167,7 +173,7 @@ class ProcessTask implements ITask { recordProcessorCheckpointer.getLargestPermittedCheckpointValue())); if (shouldCallProcessRecords(records)) { - callProcessRecords(getRecordsResult, records); + callProcessRecords(processRecordsInput, records); } } catch (ProvisionedThroughputExceededException pte) { throttlingReporter.throttled(); @@ -197,18 +203,18 @@ class ProcessTask implements ITask { /** * Dispatches a batch of records to the record processor, and handles any fallout from that. - * - * @param getRecordsResult + * + * @param input * the result of the last call to Kinesis * @param records * the records to be dispatched. It's possible the records have been adjusted by KPL deaggregation. */ - private void callProcessRecords(GetRecordsResult getRecordsResult, List records) { + private void callProcessRecords(ProcessRecordsInput input, List records) { LOG.debug("Calling application processRecords() with " + records.size() + " records from " + shardInfo.getShardId()); final ProcessRecordsInput processRecordsInput = new ProcessRecordsInput().withRecords(records) .withCheckpointer(recordProcessorCheckpointer) - .withMillisBehindLatest(getRecordsResult.getMillisBehindLatest()); + .withMillisBehindLatest(input.getMillisBehindLatest()); final long recordProcessorStartTimeMillis = System.currentTimeMillis(); try { @@ -225,7 +231,7 @@ class ProcessTask implements ITask { /** * Whether we should call process records or not - * + * * @param records * the records returned from the call to Kinesis, and/or deaggregation * @return true if the set of records should be dispatched to the record process, false if they should not. @@ -236,7 +242,7 @@ class ProcessTask implements ITask { /** * Determines whether to deaggregate the given records, and if they are KPL records dispatches them to deaggregation - * + * * @param records * the records to deaggregate is deaggregation is required. * @return returns either the deaggregated records, or the original records @@ -259,7 +265,7 @@ class ProcessTask implements ITask { /** * Emits metrics, and sleeps if there are no records available - * + * * @param startTimeMillis * the time when the task started */ @@ -296,8 +302,8 @@ class ProcessTask implements ITask { * @return the largest extended sequence number among the retained records */ private ExtendedSequenceNumber filterAndGetMaxExtendedSequenceNumber(IMetricsScope scope, List records, - final ExtendedSequenceNumber lastCheckpointValue, - final ExtendedSequenceNumber lastLargestPermittedCheckpointValue) { + final ExtendedSequenceNumber lastCheckpointValue, + final ExtendedSequenceNumber lastLargestPermittedCheckpointValue) { ExtendedSequenceNumber largestExtendedSequenceNumber = lastLargestPermittedCheckpointValue; ListIterator recordIterator = records.listIterator(); while (recordIterator.hasNext()) { @@ -331,7 +337,7 @@ class ProcessTask implements ITask { * * @return list of data records from Kinesis */ - private GetRecordsResult getRecordsResult() { + private ProcessRecordsInput getRecordsResult() { try { return getRecordsResultAndRecordMillisBehindLatest(); } catch (ExpiredIteratorException e) { @@ -367,22 +373,17 @@ class ProcessTask implements ITask { * * @return list of data records from Kinesis */ - private GetRecordsResult getRecordsResultAndRecordMillisBehindLatest() { - final GetRecordsResult getRecordsResult = dataFetcher.getRecords(streamConfig.getMaxRecords()); + private ProcessRecordsInput getRecordsResultAndRecordMillisBehindLatest() { + final ProcessRecordsInput processRecordsInput = getRecordsCache.getNextResult(); - if (getRecordsResult == null) { - // Stream no longer exists - return new GetRecordsResult().withRecords(Collections.emptyList()); - } - - if (getRecordsResult.getMillisBehindLatest() != null) { + if (processRecordsInput.getMillisBehindLatest() != null) { MetricsHelper.getMetricsScope().addData(MILLIS_BEHIND_LATEST_METRIC, - getRecordsResult.getMillisBehindLatest(), + processRecordsInput.getMillisBehindLatest(), StandardUnit.Milliseconds, MetricsLevel.SUMMARY); } - return getRecordsResult; + return processRecordsInput; } -} +} \ No newline at end of file diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/RecordProcessorCheckpointer.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/RecordProcessorCheckpointer.java index 69922670..8e3dfd73 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/RecordProcessorCheckpointer.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/RecordProcessorCheckpointer.java @@ -1,5 +1,5 @@ /* - * Copyright 2012-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Amazon Software License (the "License"). * You may not use this file except in compliance with the License. @@ -14,6 +14,9 @@ */ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; +import com.amazonaws.services.kinesis.metrics.impl.MetricsHelper; +import com.amazonaws.services.kinesis.metrics.impl.ThreadSafeMetricsDelegatingScope; +import com.amazonaws.services.kinesis.metrics.interfaces.IMetricsFactory; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -23,6 +26,7 @@ import com.amazonaws.services.kinesis.clientlibrary.exceptions.KinesisClientLibE import com.amazonaws.services.kinesis.clientlibrary.exceptions.ShutdownException; import com.amazonaws.services.kinesis.clientlibrary.exceptions.ThrottlingException; import com.amazonaws.services.kinesis.clientlibrary.interfaces.ICheckpoint; +import com.amazonaws.services.kinesis.clientlibrary.interfaces.IPreparedCheckpointer; import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer; import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber; import com.amazonaws.services.kinesis.clientlibrary.types.UserRecord; @@ -49,6 +53,8 @@ class RecordProcessorCheckpointer implements IRecordProcessorCheckpointer { private SequenceNumberValidator sequenceNumberValidator; private ExtendedSequenceNumber sequenceNumberAtShardEnd; + + private IMetricsFactory metricsFactory; /** * Only has package level access, since only the Amazon Kinesis Client Library should be creating these. @@ -58,10 +64,12 @@ class RecordProcessorCheckpointer implements IRecordProcessorCheckpointer { */ RecordProcessorCheckpointer(ShardInfo shardInfo, ICheckpoint checkpoint, - SequenceNumberValidator validator) { + SequenceNumberValidator validator, + IMetricsFactory metricsFactory) { this.shardInfo = shardInfo; this.checkpoint = checkpoint; this.sequenceNumberValidator = validator; + this.metricsFactory = metricsFactory; } /** @@ -127,7 +135,7 @@ class RecordProcessorCheckpointer implements IRecordProcessorCheckpointer { * If there is a last checkpoint value, we want to check both the lower and upper bound. */ ExtendedSequenceNumber newCheckpoint = new ExtendedSequenceNumber(sequenceNumber, subSequenceNumber); - if ((lastCheckpointValue.compareTo(newCheckpoint) <= 0) + if ((lastCheckpointValue == null || lastCheckpointValue.compareTo(newCheckpoint) <= 0) && newCheckpoint.compareTo(largestPermittedCheckpointValue) <= 0) { if (LOG.isDebugEnabled()) { @@ -144,6 +152,82 @@ class RecordProcessorCheckpointer implements IRecordProcessorCheckpointer { } } + /** + * {@inheritDoc} + */ + @Override + public synchronized IPreparedCheckpointer prepareCheckpoint() + throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException { + return this.prepareCheckpoint( + this.largestPermittedCheckpointValue.getSequenceNumber(), + this.largestPermittedCheckpointValue.getSubSequenceNumber()); + } + + /** + * {@inheritDoc} + */ + @Override + public synchronized IPreparedCheckpointer prepareCheckpoint(Record record) + throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException { + if (record == null) { + throw new IllegalArgumentException("Could not prepare checkpoint a null record"); + } else if (record instanceof UserRecord) { + return prepareCheckpoint(record.getSequenceNumber(), ((UserRecord) record).getSubSequenceNumber()); + } else { + return prepareCheckpoint(record.getSequenceNumber(), 0); + } + } + + /** + * {@inheritDoc} + */ + @Override + public synchronized IPreparedCheckpointer prepareCheckpoint(String sequenceNumber) + throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException { + return prepareCheckpoint(sequenceNumber, 0); + } + + /** + * {@inheritDoc} + */ + @Override + public synchronized IPreparedCheckpointer prepareCheckpoint(String sequenceNumber, long subSequenceNumber) + throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException { + + if (subSequenceNumber < 0) { + throw new IllegalArgumentException("Could not checkpoint at invalid, negative subsequence number " + + subSequenceNumber); + } + + // throws exception if sequence number shouldn't be checkpointed for this shard + sequenceNumberValidator.validateSequenceNumber(sequenceNumber); + if (LOG.isDebugEnabled()) { + LOG.debug("Validated prepareCheckpoint sequence number " + sequenceNumber + " for " + shardInfo.getShardId() + + ", token " + shardInfo.getConcurrencyToken()); + } + /* + * If there isn't a last checkpoint value, we only care about checking the upper bound. + * If there is a last checkpoint value, we want to check both the lower and upper bound. + */ + ExtendedSequenceNumber pendingCheckpoint = new ExtendedSequenceNumber(sequenceNumber, subSequenceNumber); + if ((lastCheckpointValue == null || lastCheckpointValue.compareTo(pendingCheckpoint) <= 0) + && pendingCheckpoint.compareTo(largestPermittedCheckpointValue) <= 0) { + + if (LOG.isDebugEnabled()) { + LOG.debug("Preparing checkpoint " + shardInfo.getShardId() + + ", token " + shardInfo.getConcurrencyToken() + + " at specific extended sequence number " + pendingCheckpoint); + } + return doPrepareCheckpoint(pendingCheckpoint); + } else { + throw new IllegalArgumentException(String.format( + "Could not prepare checkpoint at extended sequence number %s as it did not fall into acceptable " + + "range between the last checkpoint %s and the greatest extended sequence number passed " + + "to this record processor %s", + pendingCheckpoint, this.lastCheckpointValue, this.largestPermittedCheckpointValue)); + } + } + /** * @return the lastCheckpointValue */ @@ -165,7 +249,7 @@ class RecordProcessorCheckpointer implements IRecordProcessorCheckpointer { } /** - * @param checkpoint the checkpoint value to set + * @param largestPermittedCheckpointValue the largest permitted checkpoint */ synchronized void setLargestPermittedCheckpointValue(ExtendedSequenceNumber largestPermittedCheckpointValue) { this.largestPermittedCheckpointValue = largestPermittedCheckpointValue; @@ -206,22 +290,88 @@ class RecordProcessorCheckpointer implements IRecordProcessorCheckpointer { // just checkpoint at SHARD_END checkpointToRecord = ExtendedSequenceNumber.SHARD_END; } + + boolean unsetMetrics = false; // Don't checkpoint a value we already successfully checkpointed - if (extendedSequenceNumber != null && !extendedSequenceNumber.equals(lastCheckpointValue)) { - try { - if (LOG.isDebugEnabled()) { - LOG.debug("Setting " + shardInfo.getShardId() + ", token " + shardInfo.getConcurrencyToken() - + " checkpoint to " + checkpointToRecord); + try { + if (!MetricsHelper.isMetricsScopePresent()) { + MetricsHelper.setMetricsScope(new ThreadSafeMetricsDelegatingScope(metricsFactory.createMetrics())); + unsetMetrics = true; + } + if (extendedSequenceNumber != null && !extendedSequenceNumber.equals(lastCheckpointValue)) { + try { + if (LOG.isDebugEnabled()) { + LOG.debug("Setting " + shardInfo.getShardId() + ", token " + shardInfo.getConcurrencyToken() + + " checkpoint to " + checkpointToRecord); + } + checkpoint.setCheckpoint(shardInfo.getShardId(), checkpointToRecord, shardInfo.getConcurrencyToken()); + lastCheckpointValue = checkpointToRecord; + } catch (ThrottlingException | ShutdownException | InvalidStateException + | KinesisClientLibDependencyException e) { + throw e; + } catch (KinesisClientLibException e) { + LOG.warn("Caught exception setting checkpoint.", e); + throw new KinesisClientLibDependencyException("Caught exception while checkpointing", e); } - checkpoint.setCheckpoint(shardInfo.getShardId(), checkpointToRecord, shardInfo.getConcurrencyToken()); - lastCheckpointValue = checkpointToRecord; - } catch (ThrottlingException | ShutdownException | InvalidStateException - | KinesisClientLibDependencyException e) { - throw e; - } catch (KinesisClientLibException e) { - LOG.warn("Caught exception setting checkpoint.", e); - throw new KinesisClientLibDependencyException("Caught exception while checkpointing", e); + } + } finally { + if (unsetMetrics) { + MetricsHelper.unsetMetricsScope(); } } } + + /** + * This method stores the given sequenceNumber as a pending checkpooint in the lease table without overwriting the + * current checkpoint, then returns a PreparedCheckpointer that is ready to checkpoint at the given sequence number. + * + * This method does not advance lastCheckpointValue, but calls to PreparedCheckpointer.checkpoint() on the returned + * objects do. This allows customers to 'discard' prepared checkpoints by calling any of the 4 checkpoint methods on + * this class before calling PreparedCheckpointer.checkpoint(). Some examples: + * + * 1) prepareCheckpoint(snA); checkpoint(snB). // this works regardless of whether snA or snB is bigger. It discards + * the prepared checkpoint at snA. + * 2) prepareCheckpoint(snA); prepareCheckpoint(snB). // this works regardless of whether snA or snB is bigger. It + * replaces the preparedCheckpoint at snA with a new one at snB. + * 3) checkpointerA = prepareCheckpoint(snA); checkpointerB = prepareCheckpoint(snB); checkpointerB.checkpoint(); + * checkpointerA.checkpoint(); // This replaces the prepared checkpoint at snA with a new one at snB, then + * checkpoints at snB regardless of whether snA or snB is bigger. The checkpoint at snA only succeeds if snA > snB. + * + * @param extendedSequenceNumber the sequence number for the prepared checkpoint + * @return a prepared checkpointer that is ready to checkpoint at the given sequence number. + * @throws KinesisClientLibDependencyException + * @throws InvalidStateException + * @throws ThrottlingException + * @throws ShutdownException + */ + private IPreparedCheckpointer doPrepareCheckpoint(ExtendedSequenceNumber extendedSequenceNumber) + throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException { + + ExtendedSequenceNumber newPrepareCheckpoint = extendedSequenceNumber; + if (sequenceNumberAtShardEnd != null && sequenceNumberAtShardEnd.equals(extendedSequenceNumber)) { + // If we are about to checkpoint the very last sequence number for this shard, we might as well + // just checkpoint at SHARD_END + newPrepareCheckpoint = ExtendedSequenceNumber.SHARD_END; + } + + // Don't actually prepare a checkpoint if they're trying to checkpoint at the current checkpointed value. + // The only way this can happen is if they call prepareCheckpoint() in a record processor that was initialized + // AND that has not processed any records since initialization. + if (newPrepareCheckpoint.equals(lastCheckpointValue)) { + return new DoesNothingPreparedCheckpointer(newPrepareCheckpoint); + } + + try { + checkpoint.prepareCheckpoint(shardInfo.getShardId(), newPrepareCheckpoint, shardInfo.getConcurrencyToken()); + } catch (ThrottlingException | ShutdownException | InvalidStateException + | KinesisClientLibDependencyException e) { + throw e; + } catch (KinesisClientLibException e) { + LOG.warn("Caught exception setting prepareCheckpoint.", e); + throw new KinesisClientLibDependencyException("Caught exception while prepareCheckpointing", e); + } + + PreparedCheckpointer result = new PreparedCheckpointer(newPrepareCheckpoint, this); + return result; + } } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/RecordsFetcherFactory.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/RecordsFetcherFactory.java new file mode 100644 index 00000000..c1a513a9 --- /dev/null +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/RecordsFetcherFactory.java @@ -0,0 +1,76 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package com.amazonaws.services.kinesis.clientlibrary.lib.worker; + +import com.amazonaws.services.kinesis.metrics.interfaces.IMetricsFactory; + +/** + * This factory is used to create the records fetcher to retrieve data from Kinesis for a given shard. + */ +public interface RecordsFetcherFactory { + /** + * Returns a GetRecordsCache to be used for retrieving records for a given shard. + * + * @param getRecordsRetrievalStrategy GetRecordsRetrievalStrategy to be used with the GetRecordsCache + * @param shardId ShardId of the shard that the fetcher will retrieve records for + * @param metricsFactory MetricsFactory used to create metricScope + * @param maxRecords Max number of records to be returned in a single get call + * + * @return GetRecordsCache used to get records from Kinesis. + */ + GetRecordsCache createRecordsFetcher(GetRecordsRetrievalStrategy getRecordsRetrievalStrategy, String shardId, + IMetricsFactory metricsFactory, int maxRecords); + + /** + * Sets the maximum number of ProcessRecordsInput objects the GetRecordsCache can hold, before further requests are + * blocked. + * + * @param maxPendingProcessRecordsInput The maximum number of ProcessRecordsInput objects that the cache will accept + * before blocking. + */ + void setMaxPendingProcessRecordsInput(int maxPendingProcessRecordsInput); + + /** + * Sets the max byte size for the GetRecordsCache, before further requests are blocked. The byte size of the cache + * is the sum of byte size of all the ProcessRecordsInput objects in the cache at any point of time. + * + * @param maxByteSize The maximum byte size for the cache before blocking. + */ + void setMaxByteSize(int maxByteSize); + + /** + * Sets the max number of records for the GetRecordsCache can hold, before further requests are blocked. The records + * count is the sum of all records present in across all the ProcessRecordsInput objects in the cache at any point + * of time. + * + * @param maxRecordsCount The mximum number of records in the cache before blocking. + */ + void setMaxRecordsCount(int maxRecordsCount); + + /** + * Sets the dataFetchingStrategy to determine the type of GetRecordsCache to be used. + * + * @param dataFetchingStrategy Fetching strategy to be used + */ + void setDataFetchingStrategy(DataFetchingStrategy dataFetchingStrategy); + + /** + * Sets the maximum idle time between two get calls. + * + * @param idleMillisBetweenCalls Sleep millis between calls. + */ + void setIdleMillisBetweenCalls(long idleMillisBetweenCalls); + +} diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/SequenceNumberValidator.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/SequenceNumberValidator.java index 96af5f7c..8cebbf33 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/SequenceNumberValidator.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/SequenceNumberValidator.java @@ -70,12 +70,14 @@ public class SequenceNumberValidator { */ void validateSequenceNumber(String sequenceNumber) throws IllegalArgumentException, ThrottlingException, KinesisClientLibDependencyException { - if (!isDigits(sequenceNumber)) { + boolean atShardEnd = ExtendedSequenceNumber.SHARD_END.getSequenceNumber().equals(sequenceNumber); + + if (!atShardEnd && !isDigits(sequenceNumber)) { LOG.info("Sequence number must be numeric, but was " + sequenceNumber); throw new IllegalArgumentException("Sequence number must be numeric, but was " + sequenceNumber); } try { - if (validateWithGetIterator) { + if (!atShardEnd &&validateWithGetIterator) { proxy.getIterator(shardId, ShardIteratorType.AFTER_SEQUENCE_NUMBER.toString(), sequenceNumber); LOG.info("Validated sequence number " + sequenceNumber + " with shard id " + shardId); } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumer.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumer.java index 63cce40d..4a001b9b 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumer.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumer.java @@ -1,20 +1,21 @@ /* - * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; +import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.RejectedExecutionException; @@ -30,6 +31,8 @@ import com.amazonaws.services.kinesis.leases.interfaces.ILeaseManager; import com.amazonaws.services.kinesis.metrics.interfaces.IMetricsFactory; import com.google.common.annotations.VisibleForTesting; +import lombok.Getter; + /** * Responsible for consuming data records of a (specified) shard. * The instance should be shutdown when we lose the primary responsibility for a shard. @@ -41,6 +44,7 @@ class ShardConsumer { private final StreamConfig streamConfig; private final IRecordProcessor recordProcessor; + private final KinesisClientLibConfiguration config; private final RecordProcessorCheckpointer recordProcessorCheckpointer; private final ExecutorService executorService; private final ShardInfo shardInfo; @@ -57,6 +61,20 @@ class ShardConsumer { private ITask currentTask; private long currentTaskSubmitTime; private Future future; + + @Getter + private final GetRecordsCache getRecordsCache; + + private static final GetRecordsRetrievalStrategy makeStrategy(KinesisDataFetcher dataFetcher, + Optional retryGetRecordsInSeconds, + Optional maxGetRecordsThreadPool, + ShardInfo shardInfo) { + Optional getRecordsRetrievalStrategy = retryGetRecordsInSeconds.flatMap(retry -> + maxGetRecordsThreadPool.map(max -> + new AsynchronousGetRecordsRetrievalStrategy(dataFetcher, retry, max, shardInfo.getShardId()))); + + return getRecordsRetrievalStrategy.orElse(new SynchronousGetRecordsRetrievalStrategy(dataFetcher)); + } /* * Tracks current state. It is only updated via the consumeStream/shutdown APIs. Therefore we don't do @@ -75,6 +93,7 @@ class ShardConsumer { * @param streamConfig Stream configuration to use * @param checkpoint Checkpoint tracker * @param recordProcessor Record processor used to process the data records for the shard + * @param config Kinesis library configuration * @param leaseManager Used to create leases for new shards * @param parentShardPollIntervalMillis Wait for this long if parent shards are not done (or we get an exception) * @param executorService ExecutorService used to execute process tasks for this shard @@ -83,34 +102,141 @@ class ShardConsumer { */ // CHECKSTYLE:IGNORE ParameterNumber FOR NEXT 10 LINES ShardConsumer(ShardInfo shardInfo, - StreamConfig streamConfig, - ICheckpoint checkpoint, - IRecordProcessor recordProcessor, - ILeaseManager leaseManager, - long parentShardPollIntervalMillis, - boolean cleanupLeasesOfCompletedShards, - ExecutorService executorService, - IMetricsFactory metricsFactory, - long backoffTimeMillis, - boolean skipShardSyncAtWorkerInitializationIfLeasesExist) { - this.streamConfig = streamConfig; - this.recordProcessor = recordProcessor; - this.executorService = executorService; - this.shardInfo = shardInfo; - this.checkpoint = checkpoint; - this.recordProcessorCheckpointer = - new RecordProcessorCheckpointer(shardInfo, + StreamConfig streamConfig, + ICheckpoint checkpoint, + IRecordProcessor recordProcessor, + ILeaseManager leaseManager, + long parentShardPollIntervalMillis, + boolean cleanupLeasesOfCompletedShards, + ExecutorService executorService, + IMetricsFactory metricsFactory, + long backoffTimeMillis, + boolean skipShardSyncAtWorkerInitializationIfLeasesExist, + KinesisClientLibConfiguration config) { + this(shardInfo, + streamConfig, + checkpoint, + recordProcessor, + leaseManager, + parentShardPollIntervalMillis, + cleanupLeasesOfCompletedShards, + executorService, + metricsFactory, + backoffTimeMillis, + skipShardSyncAtWorkerInitializationIfLeasesExist, + Optional.empty(), + Optional.empty(), + config); + } + + /** + * @param shardInfo Shard information + * @param streamConfig Stream configuration to use + * @param checkpoint Checkpoint tracker + * @param recordProcessor Record processor used to process the data records for the shard + * @param leaseManager Used to create leases for new shards + * @param parentShardPollIntervalMillis Wait for this long if parent shards are not done (or we get an exception) + * @param executorService ExecutorService used to execute process tasks for this shard + * @param metricsFactory IMetricsFactory used to construct IMetricsScopes for this shard + * @param backoffTimeMillis backoff interval when we encounter exceptions + * @param retryGetRecordsInSeconds time in seconds to wait before the worker retries to get a record. + * @param maxGetRecordsThreadPool max number of threads in the getRecords thread pool. + * @param config Kinesis library configuration + */ + // CHECKSTYLE:IGNORE ParameterNumber FOR NEXT 10 LINES + ShardConsumer(ShardInfo shardInfo, + StreamConfig streamConfig, + ICheckpoint checkpoint, + IRecordProcessor recordProcessor, + ILeaseManager leaseManager, + long parentShardPollIntervalMillis, + boolean cleanupLeasesOfCompletedShards, + ExecutorService executorService, + IMetricsFactory metricsFactory, + long backoffTimeMillis, + boolean skipShardSyncAtWorkerInitializationIfLeasesExist, + Optional retryGetRecordsInSeconds, + Optional maxGetRecordsThreadPool, + KinesisClientLibConfiguration config) { + + this( + shardInfo, + streamConfig, + checkpoint, + recordProcessor, + new RecordProcessorCheckpointer( + shardInfo, checkpoint, - new SequenceNumberValidator(streamConfig.getStreamProxy(), + new SequenceNumberValidator( + streamConfig.getStreamProxy(), shardInfo.getShardId(), - streamConfig.shouldValidateSequenceNumberBeforeCheckpointing())); - this.dataFetcher = new KinesisDataFetcher(streamConfig.getStreamProxy(), shardInfo); + streamConfig.shouldValidateSequenceNumberBeforeCheckpointing()), + metricsFactory), + leaseManager, + parentShardPollIntervalMillis, + cleanupLeasesOfCompletedShards, + executorService, + metricsFactory, + backoffTimeMillis, + skipShardSyncAtWorkerInitializationIfLeasesExist, + new KinesisDataFetcher(streamConfig.getStreamProxy(), shardInfo), + retryGetRecordsInSeconds, + maxGetRecordsThreadPool, + config + ); + } + + /** + * @param shardInfo Shard information + * @param streamConfig Stream Config to use + * @param checkpoint Checkpoint tracker + * @param recordProcessor Record processor used to process the data records for the shard + * @param recordProcessorCheckpointer RecordProcessorCheckpointer to use to checkpoint progress + * @param leaseManager Used to create leases for new shards + * @param parentShardPollIntervalMillis Wait for this long if parent shards are not done (or we get an exception) + * @param cleanupLeasesOfCompletedShards clean up the leases of completed shards + * @param executorService ExecutorService used to execute process tasks for this shard + * @param metricsFactory IMetricsFactory used to construct IMetricsScopes for this shard + * @param backoffTimeMillis backoff interval when we encounter exceptions + * @param skipShardSyncAtWorkerInitializationIfLeasesExist Skip sync at init if lease exists + * @param kinesisDataFetcher KinesisDataFetcher to fetch data from Kinesis streams. + * @param retryGetRecordsInSeconds time in seconds to wait before the worker retries to get a record + * @param maxGetRecordsThreadPool max number of threads in the getRecords thread pool + * @param config Kinesis library configuration + */ + ShardConsumer(ShardInfo shardInfo, + StreamConfig streamConfig, + ICheckpoint checkpoint, + IRecordProcessor recordProcessor, + RecordProcessorCheckpointer recordProcessorCheckpointer, + ILeaseManager leaseManager, + long parentShardPollIntervalMillis, + boolean cleanupLeasesOfCompletedShards, + ExecutorService executorService, + IMetricsFactory metricsFactory, + long backoffTimeMillis, + boolean skipShardSyncAtWorkerInitializationIfLeasesExist, + KinesisDataFetcher kinesisDataFetcher, + Optional retryGetRecordsInSeconds, + Optional maxGetRecordsThreadPool, + KinesisClientLibConfiguration config) { + this.shardInfo = shardInfo; + this.streamConfig = streamConfig; + this.checkpoint = checkpoint; + this.recordProcessor = recordProcessor; + this.recordProcessorCheckpointer = recordProcessorCheckpointer; this.leaseManager = leaseManager; - this.metricsFactory = metricsFactory; this.parentShardPollIntervalMillis = parentShardPollIntervalMillis; this.cleanupLeasesOfCompletedShards = cleanupLeasesOfCompletedShards; + this.executorService = executorService; + this.metricsFactory = metricsFactory; this.taskBackoffTimeMillis = backoffTimeMillis; this.skipShardSyncAtWorkerInitializationIfLeasesExist = skipShardSyncAtWorkerInitializationIfLeasesExist; + this.config = config; + this.dataFetcher = kinesisDataFetcher; + this.getRecordsCache = config.getRecordsFetcherFactory().createRecordsFetcher( + makeStrategy(this.dataFetcher, retryGetRecordsInSeconds, maxGetRecordsThreadPool, this.shardInfo), + this.getShardInfo().getShardId(), this.metricsFactory, this.config.getMaxRecords()); } /** @@ -158,11 +284,17 @@ class ShardConsumer { } } } else { + final long timeElapsed = System.currentTimeMillis() - currentTaskSubmitTime; + final String commonMessage = String.format("Previous %s task still pending for shard %s since %d ms ago. ", + currentTask.getTaskType(), shardInfo.getShardId(), timeElapsed); if (LOG.isDebugEnabled()) { - LOG.debug("Previous " + currentTask.getTaskType() + " task still pending for shard " - + shardInfo.getShardId() + " since " + (System.currentTimeMillis() - currentTaskSubmitTime) - + " ms ago" + ". Not submitting new task."); + LOG.debug(commonMessage + "Not submitting new task."); } + config.getLogWarningForTaskAfterMillis().ifPresent(value -> { + if (timeElapsed > value) { + LOG.warn(commonMessage); + } + }); } return submittedNewTask; @@ -281,7 +413,7 @@ class ShardConsumer { if (taskOutcome == TaskOutcome.END_OF_SHARD) { markForShutdown(ShutdownReason.TERMINATE); } - if (isShutdownRequested()) { + if (isShutdownRequested() && taskOutcome != TaskOutcome.FAILURE) { currentState = currentState.shutdownTransition(shutdownReason); } else if (taskOutcome == TaskOutcome.SUCCESSFUL) { if (currentState.getTaskType() == currentTask.getTaskType()) { @@ -353,6 +485,10 @@ class ShardConsumer { return cleanupLeasesOfCompletedShards; } + boolean isIgnoreUnexpectedChildShards() { + return config.shouldIgnoreUnexpectedChildShards(); + } + long getTaskBackoffTimeMillis() { return taskBackoffTimeMillis; } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumerShutdownNotification.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumerShutdownNotification.java index b3792131..aa5a7942 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumerShutdownNotification.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumerShutdownNotification.java @@ -1,3 +1,17 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; import java.util.concurrent.CountDownLatch; diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardInfo.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardInfo.java index c339e9f9..00d8cc64 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardInfo.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardInfo.java @@ -1,16 +1,16 @@ /* - * Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; @@ -19,8 +19,8 @@ import java.util.Collections; import java.util.LinkedList; import java.util.List; -import org.apache.commons.lang.builder.EqualsBuilder; -import org.apache.commons.lang.builder.HashCodeBuilder; +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber; diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardPrioritization.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardPrioritization.java index 54f7517d..442c37dd 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardPrioritization.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardPrioritization.java @@ -1,3 +1,17 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; import java.util.List; diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTask.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTask.java index ddfb8459..5a0c3d5a 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTask.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTask.java @@ -35,6 +35,7 @@ class ShardSyncTask implements ITask { private final ILeaseManager leaseManager; private InitialPositionInStreamExtended initialPosition; private final boolean cleanupLeasesUponShardCompletion; + private final boolean ignoreUnexpectedChildShards; private final long shardSyncTaskIdleTimeMillis; private final TaskType taskType = TaskType.SHARDSYNC; @@ -49,11 +50,13 @@ class ShardSyncTask implements ITask { ILeaseManager leaseManager, InitialPositionInStreamExtended initialPositionInStream, boolean cleanupLeasesUponShardCompletion, + boolean ignoreUnexpectedChildShards, long shardSyncTaskIdleTimeMillis) { this.kinesisProxy = kinesisProxy; this.leaseManager = leaseManager; this.initialPosition = initialPositionInStream; this.cleanupLeasesUponShardCompletion = cleanupLeasesUponShardCompletion; + this.ignoreUnexpectedChildShards = ignoreUnexpectedChildShards; this.shardSyncTaskIdleTimeMillis = shardSyncTaskIdleTimeMillis; } @@ -68,7 +71,8 @@ class ShardSyncTask implements ITask { ShardSyncer.checkAndCreateLeasesForNewShards(kinesisProxy, leaseManager, initialPosition, - cleanupLeasesUponShardCompletion); + cleanupLeasesUponShardCompletion, + ignoreUnexpectedChildShards); if (shardSyncTaskIdleTimeMillis > 0) { Thread.sleep(shardSyncTaskIdleTimeMillis); } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTaskManager.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTaskManager.java index c1bfae76..be62c66b 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTaskManager.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTaskManager.java @@ -44,6 +44,7 @@ class ShardSyncTaskManager { private final ExecutorService executorService; private final InitialPositionInStreamExtended initialPositionInStream; private boolean cleanupLeasesUponShardCompletion; + private boolean ignoreUnexpectedChildShards; private final long shardSyncIdleTimeMillis; @@ -55,6 +56,7 @@ class ShardSyncTaskManager { * @param initialPositionInStream Initial position in stream * @param cleanupLeasesUponShardCompletion Clean up leases for shards that we've finished processing (don't wait * until they expire) + * @param ignoreUnexpectedChildShards Ignore child shards with open parents * @param shardSyncIdleTimeMillis Time between tasks to sync leases and Kinesis shards * @param metricsFactory Metrics factory * @param executorService ExecutorService to execute the shard sync tasks @@ -63,6 +65,7 @@ class ShardSyncTaskManager { final ILeaseManager leaseManager, final InitialPositionInStreamExtended initialPositionInStream, final boolean cleanupLeasesUponShardCompletion, + final boolean ignoreUnexpectedChildShards, final long shardSyncIdleTimeMillis, final IMetricsFactory metricsFactory, ExecutorService executorService) { @@ -70,6 +73,7 @@ class ShardSyncTaskManager { this.leaseManager = leaseManager; this.metricsFactory = metricsFactory; this.cleanupLeasesUponShardCompletion = cleanupLeasesUponShardCompletion; + this.ignoreUnexpectedChildShards = ignoreUnexpectedChildShards; this.shardSyncIdleTimeMillis = shardSyncIdleTimeMillis; this.executorService = executorService; this.initialPositionInStream = initialPositionInStream; @@ -99,6 +103,7 @@ class ShardSyncTaskManager { leaseManager, initialPositionInStream, cleanupLeasesUponShardCompletion, + ignoreUnexpectedChildShards, shardSyncIdleTimeMillis), metricsFactory); future = executorService.submit(currentTask); submittedNewTask = true; diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncer.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncer.java index 52944200..3194cd41 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncer.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncer.java @@ -28,6 +28,7 @@ import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.commons.lang3.StringUtils; import com.amazonaws.services.kinesis.clientlibrary.exceptions.internal.KinesisClientLibIOException; import com.amazonaws.services.kinesis.clientlibrary.proxies.IKinesisProxy; @@ -60,9 +61,11 @@ class ShardSyncer { static synchronized void bootstrapShardLeases(IKinesisProxy kinesisProxy, ILeaseManager leaseManager, InitialPositionInStreamExtended initialPositionInStream, - boolean cleanupLeasesOfCompletedShards) + boolean cleanupLeasesOfCompletedShards, + boolean ignoreUnexpectedChildShards) throws DependencyException, InvalidStateException, ProvisionedThroughputException, KinesisClientLibIOException { - syncShardLeases(kinesisProxy, leaseManager, initialPositionInStream, cleanupLeasesOfCompletedShards); + syncShardLeases(kinesisProxy, leaseManager, initialPositionInStream, cleanupLeasesOfCompletedShards, + ignoreUnexpectedChildShards); } /** @@ -71,21 +74,28 @@ class ShardSyncer { * @param kinesisProxy * @param leaseManager * @param initialPositionInStream - * @param expectedClosedShardId If this is not null, we will assert that the shard list we get from Kinesis - * shows this shard to be closed (e.g. parent shard must be closed after a reshard operation). - * If it is open, we assume this is an race condition around a reshard event and throw - * a KinesisClientLibIOException so client can backoff and retry later. + * @param cleanupLeasesOfCompletedShards + * @param ignoreUnexpectedChildShards * @throws DependencyException * @throws InvalidStateException * @throws ProvisionedThroughputException * @throws KinesisClientLibIOException */ + static synchronized void checkAndCreateLeasesForNewShards(IKinesisProxy kinesisProxy, + ILeaseManager leaseManager, + InitialPositionInStreamExtended initialPositionInStream, + boolean cleanupLeasesOfCompletedShards, + boolean ignoreUnexpectedChildShards) + throws DependencyException, InvalidStateException, ProvisionedThroughputException, KinesisClientLibIOException { + syncShardLeases(kinesisProxy, leaseManager, initialPositionInStream, cleanupLeasesOfCompletedShards, ignoreUnexpectedChildShards); + } + static synchronized void checkAndCreateLeasesForNewShards(IKinesisProxy kinesisProxy, ILeaseManager leaseManager, InitialPositionInStreamExtended initialPositionInStream, boolean cleanupLeasesOfCompletedShards) throws DependencyException, InvalidStateException, ProvisionedThroughputException, KinesisClientLibIOException { - syncShardLeases(kinesisProxy, leaseManager, initialPositionInStream, cleanupLeasesOfCompletedShards); + checkAndCreateLeasesForNewShards(kinesisProxy, leaseManager, initialPositionInStream, cleanupLeasesOfCompletedShards, false); } /** @@ -93,11 +103,9 @@ class ShardSyncer { * * @param kinesisProxy * @param leaseManager - * @param expectedClosedShardId If this is not null, we will assert that the shard list we get from Kinesis - * does not show this shard to be open (e.g. parent shard must be closed after a reshard operation). - * If it is still open, we assume this is a race condition around a reshard event and - * throw a KinesisClientLibIOException so client can backoff and retry later. If the shard doesn't exist in - * Kinesis at all, we assume this is an old/expired shard and continue with the sync operation. + * @param initialPosition + * @param cleanupLeasesOfCompletedShards + * @param ignoreUnexpectedChildShards * @throws DependencyException * @throws InvalidStateException * @throws ProvisionedThroughputException @@ -107,18 +115,23 @@ class ShardSyncer { private static synchronized void syncShardLeases(IKinesisProxy kinesisProxy, ILeaseManager leaseManager, InitialPositionInStreamExtended initialPosition, - boolean cleanupLeasesOfCompletedShards) + boolean cleanupLeasesOfCompletedShards, + boolean ignoreUnexpectedChildShards) throws DependencyException, InvalidStateException, ProvisionedThroughputException, KinesisClientLibIOException { List shards = getShardList(kinesisProxy); LOG.debug("Num shards: " + shards.size()); Map shardIdToShardMap = constructShardIdToShardMap(shards); Map> shardIdToChildShardIdsMap = constructShardIdToChildShardIdsMap(shardIdToShardMap); - assertAllParentShardsAreClosed(shardIdToChildShardIdsMap, shardIdToShardMap); - + Set inconsistentShardIds = findInconsistentShardIds(shardIdToChildShardIdsMap, shardIdToShardMap); + if (!ignoreUnexpectedChildShards) { + assertAllParentShardsAreClosed(inconsistentShardIds); + } + List currentLeases = leaseManager.listLeases(); - - List newLeasesToCreate = determineNewLeasesToCreate(shards, currentLeases, initialPosition); + + List newLeasesToCreate = determineNewLeasesToCreate(shards, currentLeases, initialPosition, + inconsistentShardIds); LOG.debug("Num new leases to create: " + newLeasesToCreate.size()); for (KinesisClientLease lease : newLeasesToCreate) { long startTimeMillis = System.currentTimeMillis(); @@ -149,19 +162,37 @@ class ShardSyncer { /** Helper method to detect a race condition between fetching the shards via paginated DescribeStream calls * and a reshard operation. - * @param shardIdToChildShardIdsMap - * @param shardIdToShardMap + * @param inconsistentShardIds * @throws KinesisClientLibIOException */ - private static void assertAllParentShardsAreClosed(Map> shardIdToChildShardIdsMap, - Map shardIdToShardMap) throws KinesisClientLibIOException { + private static void assertAllParentShardsAreClosed(Set inconsistentShardIds) + throws KinesisClientLibIOException { + if (!inconsistentShardIds.isEmpty()) { + String ids = StringUtils.join(inconsistentShardIds, ' '); + throw new KinesisClientLibIOException(String.format("%d open child shards (%s) are inconsistent. " + + "This can happen due to a race condition between describeStream and a reshard operation.", + inconsistentShardIds.size(), ids)); + } + } + + /** + * Helper method to construct the list of inconsistent shards, which are open shards with non-closed ancestor + * parent(s). + * @param shardIdToChildShardIdsMap + * @param shardIdToShardMap + * @return Set of inconsistent open shard ids for shards having open parents. + */ + private static Set findInconsistentShardIds(Map> shardIdToChildShardIdsMap, + Map shardIdToShardMap) { + Set result = new HashSet(); for (String parentShardId : shardIdToChildShardIdsMap.keySet()) { Shard parentShard = shardIdToShardMap.get(parentShardId); if ((parentShardId == null) || (parentShard.getSequenceNumberRange().getEndingSequenceNumber() == null)) { - throw new KinesisClientLibIOException("Parent shardId " + parentShardId + " is not closed. " - + "This can happen due to a race condition between describeStream and a reshard operation."); + Set childShardIdsMap = shardIdToChildShardIdsMap.get(parentShardId); + result.addAll(childShardIdsMap); } } + return result; } /** @@ -296,8 +327,8 @@ class ShardSyncer { /** * Determine new leases to create and their initial checkpoint. * Note: Package level access only for testing purposes. - * - * For each open (no ending sequence number) shard that doesn't already have a lease, + * + * For each open (no ending sequence number) shard without open parents that doesn't already have a lease, * determine if it is a descendent of any shard which is or will be processed (e.g. for which a lease exists): * If so, set checkpoint of the shard to TrimHorizon and also create leases for ancestors if needed. * If not, set checkpoint of the shard to the initial position specified by the client. @@ -315,27 +346,35 @@ class ShardSyncer { * * For example: * Shard structure (each level depicts a stream segment): - * 0 1 2 3 4 5- shards till epoch 102 - * \ / \ / | | - * 6 7 4 5- shards from epoch 103 - 205 - * \ / | /\ - * 8 4 9 10 - shards from epoch 206 (open - no ending sequenceNumber) + * 0 1 2 3 4 5 - shards till epoch 102 + * \ / \ / | | + * 6 7 4 5 - shards from epoch 103 - 205 + * \ / | / \ + * 8 4 9 10 - shards from epoch 206 (open - no ending sequenceNumber) * Current leases: (3, 4, 5) * New leases to create: (2, 6, 7, 8, 9, 10) * * The leases returned are sorted by the starting sequence number - following the same order * when persisting the leases in DynamoDB will ensure that we recover gracefully if we fail * before creating all the leases. + * + * If a shard has no existing lease, is open, and is a descendant of a parent which is still open, we ignore it + * here; this happens when the list of shards is inconsistent, which could be due to pagination delay for very + * high shard count streams (i.e., dynamodb streams for tables with thousands of partitions). This can only + * currently happen here if ignoreUnexpectedChildShards was true in syncShardleases. + * * * @param shards List of all shards in Kinesis (we'll create new leases based on this set) * @param currentLeases List of current leases * @param initialPosition One of LATEST, TRIM_HORIZON, or AT_TIMESTAMP. We'll start fetching records from that * location in the shard (when an application starts up for the first time - and there are no checkpoints). + * @param inconsistentShardIds Set of child shard ids having open parents. * @return List of new leases to create sorted by starting sequenceNumber of the corresponding shard */ static List determineNewLeasesToCreate(List shards, List currentLeases, - InitialPositionInStreamExtended initialPosition) { + InitialPositionInStreamExtended initialPosition, + Set inconsistentShardIds) { Map shardIdToNewLeaseMap = new HashMap(); Map shardIdToShardMapOfAllKinesisShards = constructShardIdToShardMap(shards); @@ -354,6 +393,8 @@ class ShardSyncer { LOG.debug("Evaluating leases for open shard " + shardId + " and its ancestors."); if (shardIdsOfCurrentLeases.contains(shardId)) { LOG.debug("Lease for shardId " + shardId + " already exists. Not creating a lease"); + } else if (inconsistentShardIds.contains(shardId)) { + LOG.info("shardId " + shardId + " is an inconsistent child. Not creating a lease"); } else { LOG.debug("Need to create a lease for shardId " + shardId); KinesisClientLease newLease = newKCLLease(shard); @@ -407,6 +448,17 @@ class ShardSyncer { return newLeasesToCreate; } + /** + * Determine new leases to create and their initial checkpoint. + * Note: Package level access only for testing purposes. + */ + static List determineNewLeasesToCreate(List shards, + List currentLeases, + InitialPositionInStreamExtended initialPosition) { + Set inconsistentShardIds = new HashSet(); + return determineNewLeasesToCreate(shards, currentLeases, initialPosition, inconsistentShardIds); + } + /** * Note: Package level access for testing purposes only. * Check if this shard is a descendant of a shard that is (or will be) processed. diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownFuture.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownFuture.java deleted file mode 100644 index 8ee96537..00000000 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownFuture.java +++ /dev/null @@ -1,155 +0,0 @@ -package com.amazonaws.services.kinesis.clientlibrary.lib.worker; - -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -/** - * Used as a response from the {@link Worker#requestShutdown()} to allow callers to wait until shutdown is complete. - */ -class ShutdownFuture implements Future { - - private static final Log log = LogFactory.getLog(ShutdownFuture.class); - - private final CountDownLatch shutdownCompleteLatch; - private final CountDownLatch notificationCompleteLatch; - private final Worker worker; - - ShutdownFuture(CountDownLatch shutdownCompleteLatch, CountDownLatch notificationCompleteLatch, Worker worker) { - this.shutdownCompleteLatch = shutdownCompleteLatch; - this.notificationCompleteLatch = notificationCompleteLatch; - this.worker = worker; - } - - @Override - public boolean cancel(boolean mayInterruptIfRunning) { - throw new UnsupportedOperationException("Cannot cancel a shutdown process"); - } - - @Override - public boolean isCancelled() { - return false; - } - - @Override - public boolean isDone() { - return isWorkerShutdownComplete(); - } - - private boolean isWorkerShutdownComplete() { - return worker.isShutdownComplete() || worker.getShardInfoShardConsumerMap().isEmpty(); - } - - private long outstandingRecordProcessors(long timeout, TimeUnit unit) - throws InterruptedException, ExecutionException, TimeoutException { - - final long startNanos = System.nanoTime(); - - // - // Awaiting for all ShardConsumer/RecordProcessors to be notified that a shutdown has been requested. - // There is the possibility of a race condition where a lease is terminated after the shutdown request - // notification is started, but before the ShardConsumer is sent the notification. In this case the - // ShardConsumer would start the lease loss shutdown, and may never call the notification methods. - // - if (!notificationCompleteLatch.await(timeout, unit)) { - long awaitingNotification = notificationCompleteLatch.getCount(); - long awaitingFinalShutdown = shutdownCompleteLatch.getCount(); - log.info("Awaiting " + awaitingNotification + " record processors to complete shutdown notification, and " - + awaitingFinalShutdown + " awaiting final shutdown"); - if (awaitingFinalShutdown != 0) { - // - // The number of record processor awaiting final shutdown should be a superset of the those awaiting - // notification - // - return checkWorkerShutdownMiss(awaitingFinalShutdown); - } - } - - long remaining = remainingTimeout(timeout, unit, startNanos); - throwTimeoutMessageIfExceeded(remaining, "Notification hasn't completed within timeout time."); - - // - // Once all record processors have been notified of the shutdown it is safe to allow the worker to - // start its shutdown behavior. Once shutdown starts it will stop renewer, and drop any remaining leases. - // - worker.shutdown(); - remaining = remainingTimeout(timeout, unit, startNanos); - throwTimeoutMessageIfExceeded(remaining, "Shutdown hasn't completed within timeout time."); - - // - // Want to wait for all the remaining ShardConsumers/RecordProcessor's to complete their final shutdown - // processing. This should really be a no-op since as part of the notification completion the lease for - // ShardConsumer is terminated. - // - if (!shutdownCompleteLatch.await(remaining, TimeUnit.NANOSECONDS)) { - long outstanding = shutdownCompleteLatch.getCount(); - log.info("Awaiting " + outstanding + " record processors to complete final shutdown"); - - return checkWorkerShutdownMiss(outstanding); - } - return 0; - } - - private long remainingTimeout(long timeout, TimeUnit unit, long startNanos) { - long checkNanos = System.nanoTime() - startNanos; - return unit.toNanos(timeout) - checkNanos; - } - - private void throwTimeoutMessageIfExceeded(long remainingNanos, String message) throws TimeoutException { - if (remainingNanos <= 0) { - throw new TimeoutException(message); - } - } - - /** - * This checks to see if the worker has already hit it's shutdown target, while there is outstanding record - * processors. This maybe a little racy due to when the value of outstanding is retrieved. In general though the - * latch should be decremented before the shutdown completion. - * - * @param outstanding - * the number of record processor still awaiting shutdown. - * @return the number of record processors awaiting shutdown, or 0 if the worker believes it's shutdown already. - */ - private long checkWorkerShutdownMiss(long outstanding) { - if (isWorkerShutdownComplete()) { - if (outstanding != 0) { - log.info("Shutdown completed, but shutdownCompleteLatch still had outstanding " + outstanding - + " with a current value of " + shutdownCompleteLatch.getCount() + ". shutdownComplete: " - + worker.isShutdownComplete() + " -- Consumer Map: " - + worker.getShardInfoShardConsumerMap().size()); - } - return 0; - } - return outstanding; - } - - @Override - public Void get() throws InterruptedException, ExecutionException { - boolean complete = false; - do { - try { - long outstanding = outstandingRecordProcessors(1, TimeUnit.SECONDS); - complete = outstanding == 0; - log.info("Awaiting " + outstanding + " consumer(s) to finish shutdown."); - } catch (TimeoutException te) { - log.info("Timeout while waiting for completion: " + te.getMessage()); - } - - } while(!complete); - return null; - } - - @Override - public Void get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { - long outstanding = outstandingRecordProcessors(timeout, unit); - if (outstanding != 0) { - throw new TimeoutException("Awaiting " + outstanding + " record processors to shutdown."); - } - return null; - } -} diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownNotification.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownNotification.java index 928e6900..8fd492cf 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownNotification.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownNotification.java @@ -1,3 +1,17 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; import com.amazonaws.services.kinesis.clientlibrary.types.ShutdownInput; diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownNotificationTask.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownNotificationTask.java index a689ee43..11997367 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownNotificationTask.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownNotificationTask.java @@ -1,3 +1,17 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer; diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownReason.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownReason.java index 8d0dfc80..05925120 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownReason.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownReason.java @@ -1,16 +1,16 @@ /* - * Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTask.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTask.java index d40fbb0e..a407f009 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTask.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTask.java @@ -1,16 +1,16 @@ /* - * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; @@ -44,22 +44,26 @@ class ShutdownTask implements ITask { private final ILeaseManager leaseManager; private final InitialPositionInStreamExtended initialPositionInStream; private final boolean cleanupLeasesOfCompletedShards; + private final boolean ignoreUnexpectedChildShards; private final TaskType taskType = TaskType.SHUTDOWN; private final long backoffTimeMillis; + private final GetRecordsCache getRecordsCache; /** * Constructor. */ // CHECKSTYLE:IGNORE ParameterNumber FOR NEXT 10 LINES ShutdownTask(ShardInfo shardInfo, - IRecordProcessor recordProcessor, - RecordProcessorCheckpointer recordProcessorCheckpointer, - ShutdownReason reason, - IKinesisProxy kinesisProxy, - InitialPositionInStreamExtended initialPositionInStream, - boolean cleanupLeasesOfCompletedShards, - ILeaseManager leaseManager, - long backoffTimeMillis) { + IRecordProcessor recordProcessor, + RecordProcessorCheckpointer recordProcessorCheckpointer, + ShutdownReason reason, + IKinesisProxy kinesisProxy, + InitialPositionInStreamExtended initialPositionInStream, + boolean cleanupLeasesOfCompletedShards, + boolean ignoreUnexpectedChildShards, + ILeaseManager leaseManager, + long backoffTimeMillis, + GetRecordsCache getRecordsCache) { this.shardInfo = shardInfo; this.recordProcessor = recordProcessor; this.recordProcessorCheckpointer = recordProcessorCheckpointer; @@ -67,8 +71,10 @@ class ShutdownTask implements ITask { this.kinesisProxy = kinesisProxy; this.initialPositionInStream = initialPositionInStream; this.cleanupLeasesOfCompletedShards = cleanupLeasesOfCompletedShards; + this.ignoreUnexpectedChildShards = ignoreUnexpectedChildShards; this.leaseManager = leaseManager; this.backoffTimeMillis = backoffTimeMillis; + this.getRecordsCache = getRecordsCache; } /* @@ -79,7 +85,7 @@ class ShutdownTask implements ITask { */ @Override public TaskResult call() { - Exception exception = null; + Exception exception; boolean applicationException = false; try { @@ -107,6 +113,8 @@ class ShutdownTask implements ITask { + shardInfo.getShardId()); } } + LOG.debug("Shutting down retrieval strategy."); + getRecordsCache.shutdown(); LOG.debug("Record processor completed shutdown() for shard " + shardInfo.getShardId()); } catch (Exception e) { applicationException = true; @@ -122,7 +130,8 @@ class ShutdownTask implements ITask { ShardSyncer.checkAndCreateLeasesForNewShards(kinesisProxy, leaseManager, initialPositionInStream, - cleanupLeasesOfCompletedShards); + cleanupLeasesOfCompletedShards, + ignoreUnexpectedChildShards); LOG.debug("Finished checking for child shards of shard " + shardInfo.getShardId()); } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/SimpleRecordsFetcherFactory.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/SimpleRecordsFetcherFactory.java new file mode 100644 index 00000000..79ad9f55 --- /dev/null +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/SimpleRecordsFetcherFactory.java @@ -0,0 +1,74 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package com.amazonaws.services.kinesis.clientlibrary.lib.worker; + +import java.util.concurrent.Executors; + +import com.amazonaws.services.kinesis.metrics.interfaces.IMetricsFactory; +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +import lombok.extern.apachecommons.CommonsLog; + +@CommonsLog +public class SimpleRecordsFetcherFactory implements RecordsFetcherFactory { + private int maxPendingProcessRecordsInput = 3; + private int maxByteSize = 8 * 1024 * 1024; + private int maxRecordsCount = 30000; + private long idleMillisBetweenCalls = 1500L; + private DataFetchingStrategy dataFetchingStrategy = DataFetchingStrategy.DEFAULT; + + @Override + public GetRecordsCache createRecordsFetcher(GetRecordsRetrievalStrategy getRecordsRetrievalStrategy, String shardId, + IMetricsFactory metricsFactory, int maxRecords) { + if(dataFetchingStrategy.equals(DataFetchingStrategy.DEFAULT)) { + return new BlockingGetRecordsCache(maxRecords, getRecordsRetrievalStrategy); + } else { + return new PrefetchGetRecordsCache(maxPendingProcessRecordsInput, maxByteSize, maxRecordsCount, maxRecords, + getRecordsRetrievalStrategy, + Executors.newFixedThreadPool(1, new ThreadFactoryBuilder() + .setDaemon(true) + .setNameFormat("prefetch-cache-" + shardId + "-%04d") + .build()), + idleMillisBetweenCalls, + metricsFactory, + "ProcessTask", + shardId); + } + } + + @Override + public void setMaxPendingProcessRecordsInput(int maxPendingProcessRecordsInput){ + this.maxPendingProcessRecordsInput = maxPendingProcessRecordsInput; + } + + @Override + public void setMaxByteSize(int maxByteSize){ + this.maxByteSize = maxByteSize; + } + + @Override + public void setMaxRecordsCount(int maxRecordsCount) { + this.maxRecordsCount = maxRecordsCount; + } + + @Override + public void setDataFetchingStrategy(DataFetchingStrategy dataFetchingStrategy){ + this.dataFetchingStrategy = dataFetchingStrategy; + } + + public void setIdleMillisBetweenCalls(final long idleMillisBetweenCalls) { + this.idleMillisBetweenCalls = idleMillisBetweenCalls; + } +} diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/SynchronousGetRecordsRetrievalStrategy.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/SynchronousGetRecordsRetrievalStrategy.java new file mode 100644 index 00000000..f4209189 --- /dev/null +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/SynchronousGetRecordsRetrievalStrategy.java @@ -0,0 +1,50 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package com.amazonaws.services.kinesis.clientlibrary.lib.worker; + +import com.amazonaws.services.kinesis.model.GetRecordsResult; +import lombok.Data; +import lombok.NonNull; + +/** + * + */ +@Data +public class SynchronousGetRecordsRetrievalStrategy implements GetRecordsRetrievalStrategy { + @NonNull + private final KinesisDataFetcher dataFetcher; + + @Override + public GetRecordsResult getRecords(final int maxRecords) { + return dataFetcher.getRecords(maxRecords).accept(); + } + + @Override + public void shutdown() { + // + // Does nothing as this retriever doesn't manage any resources + // + } + + @Override + public boolean isShutdown() { + return false; + } + + @Override + public KinesisDataFetcher getDataFetcher() { + return dataFetcher; + } +} diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ThrottlingReporter.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ThrottlingReporter.java index f88f131f..f80bdd29 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ThrottlingReporter.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ThrottlingReporter.java @@ -1,3 +1,17 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; import lombok.Getter; diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java index f3bf1507..15076005 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java @@ -1,91 +1,111 @@ /* - * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; import java.util.Collection; import java.util.HashSet; import java.util.List; +import java.util.Optional; import java.util.Set; +import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Consumer; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import com.amazonaws.regions.Region; +import com.amazonaws.AmazonWebServiceClient; +import com.amazonaws.ClientConfiguration; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.regions.RegionUtils; +import com.amazonaws.regions.Regions; import com.amazonaws.services.cloudwatch.AmazonCloudWatch; import com.amazonaws.services.cloudwatch.AmazonCloudWatchClient; +import com.amazonaws.services.cloudwatch.AmazonCloudWatchClientBuilder; import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; +import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder; import com.amazonaws.services.kinesis.AmazonKinesis; import com.amazonaws.services.kinesis.AmazonKinesisClient; +import com.amazonaws.services.kinesis.AmazonKinesisClientBuilder; import com.amazonaws.services.kinesis.clientlibrary.interfaces.ICheckpoint; import com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessor; import com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessorFactory; import com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IShutdownNotificationAware; -import com.amazonaws.services.kinesis.clientlibrary.proxies.KinesisProxyFactory; +import com.amazonaws.services.kinesis.clientlibrary.proxies.IKinesisProxy; +import com.amazonaws.services.kinesis.clientlibrary.proxies.KinesisProxy; import com.amazonaws.services.kinesis.leases.exceptions.LeasingException; import com.amazonaws.services.kinesis.leases.impl.KinesisClientLease; import com.amazonaws.services.kinesis.leases.impl.KinesisClientLeaseManager; +import com.amazonaws.services.kinesis.leases.interfaces.ILeaseManager; import com.amazonaws.services.kinesis.metrics.impl.CWMetricsFactory; import com.amazonaws.services.kinesis.metrics.impl.NullMetricsFactory; import com.amazonaws.services.kinesis.metrics.interfaces.IMetricsFactory; import com.amazonaws.services.kinesis.metrics.interfaces.MetricsLevel; import com.google.common.annotations.VisibleForTesting; -import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import lombok.NonNull; +import lombok.Setter; +import lombok.experimental.Accessors; + /** - * Worker is the high level class that Kinesis applications use to start - * processing data. It initializes and oversees different components (e.g. - * syncing shard and lease information, tracking shard assignments, and - * processing data from the shards). + * Worker is the high level class that Kinesis applications use to start processing data. It initializes and oversees + * different components (e.g. syncing shard and lease information, tracking shard assignments, and processing data from + * the shards). */ public class Worker implements Runnable { private static final Log LOG = LogFactory.getLog(Worker.class); private static final int MAX_INITIALIZATION_ATTEMPTS = 20; - private static final int MAX_RETRIES = 4; + private static final WorkerStateChangeListener DEFAULT_WORKER_STATE_CHANGE_LISTENER = new NoOpWorkerStateChangeListener(); private WorkerLog wlog = new WorkerLog(); private final String applicationName; private final IRecordProcessorFactory recordProcessorFactory; + private final KinesisClientLibConfiguration config; private final StreamConfig streamConfig; private final InitialPositionInStreamExtended initialPosition; private final ICheckpoint checkpointTracker; private final long idleTimeInMilliseconds; // Backoff time when polling to check if application has finished processing // parent shards - private final long parentShardPollIntervalMillis; + private final long parentShardPollIntervalMillis; private final ExecutorService executorService; private final IMetricsFactory metricsFactory; // Backoff time when running tasks if they encounter exceptions private final long taskBackoffTimeMillis; private final long failoverTimeMillis; - // private final KinesisClientLeaseManager leaseManager; + private final Optional retryGetRecordsInSeconds; + private final Optional maxGetRecordsThreadPool; + private final KinesisClientLibLeaseCoordinator leaseCoordinator; private final ShardSyncTaskManager controlServer; @@ -97,18 +117,34 @@ public class Worker implements Runnable { // Holds consumers for shards the worker is currently tracking. Key is shard // info, value is ShardConsumer. - private ConcurrentMap shardInfoShardConsumerMap = - new ConcurrentHashMap(); + private ConcurrentMap shardInfoShardConsumerMap = new ConcurrentHashMap(); private final boolean cleanupLeasesUponShardCompletion; - + private final boolean skipShardSyncAtWorkerInitializationIfLeasesExist; + /** + * Used to ensure that only one requestedShutdown is in progress at a time. + */ + private Future gracefulShutdownFuture; + @VisibleForTesting + protected boolean gracefuleShutdownStarted = false; + @VisibleForTesting + protected GracefulShutdownCoordinator gracefulShutdownCoordinator = new GracefulShutdownCoordinator(); + + private WorkerStateChangeListener workerStateChangeListener; + /** * Constructor. * - * @param recordProcessorFactory Used to get record processor instances for processing data from shards - * @param config Kinesis Client Library configuration + * @deprecated The access to this constructor will be changed in a future release. The recommended way to create + * a Worker is to use {@link Builder} + * + * @param recordProcessorFactory + * Used to get record processor instances for processing data from shards + * @param config + * Kinesis Client Library configuration */ + @Deprecated public Worker( com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration config) { @@ -118,120 +154,238 @@ public class Worker implements Runnable { /** * Constructor. * - * @param recordProcessorFactory Used to get record processor instances for processing data from shards - * @param config Kinesis Client Library configuration - * @param execService ExecutorService to use for processing records (support for multi-threaded - * consumption) + * @deprecated The access to this constructor will be changed in a future release. The recommended way to create + * a Worker is to use {@link Builder} + * + * @param recordProcessorFactory + * Used to get record processor instances for processing data from shards + * @param config + * Kinesis Client Library configuration + * @param execService + * ExecutorService to use for processing records (support for multi-threaded consumption) */ + @Deprecated public Worker( com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorFactory recordProcessorFactory, - KinesisClientLibConfiguration config, - ExecutorService execService) { - this(recordProcessorFactory, config, new AmazonKinesisClient(config.getKinesisCredentialsProvider(), - config.getKinesisClientConfiguration()), + KinesisClientLibConfiguration config, ExecutorService execService) { + this(recordProcessorFactory, config, + new AmazonKinesisClient(config.getKinesisCredentialsProvider(), config.getKinesisClientConfiguration()), new AmazonDynamoDBClient(config.getDynamoDBCredentialsProvider(), config.getDynamoDBClientConfiguration()), new AmazonCloudWatchClient(config.getCloudWatchCredentialsProvider(), - config.getCloudWatchClientConfiguration()), execService); + config.getCloudWatchClientConfiguration()), + execService); } /** - * @param recordProcessorFactory Used to get record processor instances for processing data from shards - * @param config Kinesis Client Library configuration - * @param metricsFactory Metrics factory used to emit metrics + * @deprecated The access to this constructor will be changed in a future release. The recommended way to create + * a Worker is to use {@link Builder} + * + * @param recordProcessorFactory + * Used to get record processor instances for processing data from shards + * @param config + * Kinesis Client Library configuration + * @param metricsFactory + * Metrics factory used to emit metrics */ + @Deprecated public Worker( com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorFactory recordProcessorFactory, - KinesisClientLibConfiguration config, - IMetricsFactory metricsFactory) { + KinesisClientLibConfiguration config, IMetricsFactory metricsFactory) { this(recordProcessorFactory, config, metricsFactory, getExecutorService()); } /** - * @param recordProcessorFactory Used to get record processor instances for processing data from shards - * @param config Kinesis Client Library configuration - * @param metricsFactory Metrics factory used to emit metrics - * @param execService ExecutorService to use for processing records (support for multi-threaded - * consumption) + * @deprecated The access to this constructor will be changed in a future release. The recommended way to create + * a Worker is to use {@link Builder} + * + * @param recordProcessorFactory + * Used to get record processor instances for processing data from shards + * @param config + * Kinesis Client Library configuration + * @param metricsFactory + * Metrics factory used to emit metrics + * @param execService + * ExecutorService to use for processing records (support for multi-threaded consumption) */ + @Deprecated public Worker( com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorFactory recordProcessorFactory, - KinesisClientLibConfiguration config, - IMetricsFactory metricsFactory, - ExecutorService execService) { - this(recordProcessorFactory, config, new AmazonKinesisClient(config.getKinesisCredentialsProvider(), - config.getKinesisClientConfiguration()), + KinesisClientLibConfiguration config, IMetricsFactory metricsFactory, ExecutorService execService) { + this(recordProcessorFactory, config, + new AmazonKinesisClient(config.getKinesisCredentialsProvider(), config.getKinesisClientConfiguration()), new AmazonDynamoDBClient(config.getDynamoDBCredentialsProvider(), - config.getDynamoDBClientConfiguration()), metricsFactory, execService); + config.getDynamoDBClientConfiguration()), + metricsFactory, execService); } /** - * @param recordProcessorFactory Used to get record processor instances for processing data from shards - * @param config Kinesis Client Library configuration - * @param kinesisClient Kinesis Client used for fetching data - * @param dynamoDBClient DynamoDB client used for checkpoints and tracking leases - * @param cloudWatchClient CloudWatch Client for publishing metrics + * @deprecated The access to this constructor will be changed in a future release. The recommended way to create + * a Worker is to use {@link Builder} + * + * @param recordProcessorFactory + * Used to get record processor instances for processing data from shards + * @param config + * Kinesis Client Library configuration + * @param kinesisClient + * Kinesis Client used for fetching data + * @param dynamoDBClient + * DynamoDB client used for checkpoints and tracking leases + * @param cloudWatchClient + * CloudWatch Client for publishing metrics */ + @Deprecated public Worker( com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorFactory recordProcessorFactory, - KinesisClientLibConfiguration config, - AmazonKinesis kinesisClient, - AmazonDynamoDB dynamoDBClient, + KinesisClientLibConfiguration config, AmazonKinesis kinesisClient, AmazonDynamoDB dynamoDBClient, AmazonCloudWatch cloudWatchClient) { this(recordProcessorFactory, config, kinesisClient, dynamoDBClient, cloudWatchClient, getExecutorService()); } /** - * @param recordProcessorFactory Used to get record processor instances for processing data from shards - * @param config Kinesis Client Library configuration - * @param kinesisClient Kinesis Client used for fetching data - * @param dynamoDBClient DynamoDB client used for checkpoints and tracking leases - * @param cloudWatchClient CloudWatch Client for publishing metrics - * @param execService ExecutorService to use for processing records (support for multi-threaded - * consumption) + * @deprecated The access to this constructor will be changed in a future release. The recommended way to create + * a Worker is to use {@link Builder} + * + * @param recordProcessorFactory + * Used to get record processor instances for processing data from shards + * @param config + * Kinesis Client Library configuration + * @param kinesisClient + * Kinesis Client used for fetching data + * @param dynamoDBClient + * DynamoDB client used for checkpoints and tracking leases + * @param cloudWatchClient + * CloudWatch Client for publishing metrics + * @param execService + * ExecutorService to use for processing records (support for multi-threaded consumption) */ + @Deprecated public Worker( com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorFactory recordProcessorFactory, - KinesisClientLibConfiguration config, - AmazonKinesis kinesisClient, - AmazonDynamoDB dynamoDBClient, - AmazonCloudWatch cloudWatchClient, - ExecutorService execService) { - this(recordProcessorFactory, config, kinesisClient, dynamoDBClient, - getMetricsFactory(cloudWatchClient, config), execService); + KinesisClientLibConfiguration config, AmazonKinesis kinesisClient, AmazonDynamoDB dynamoDBClient, + AmazonCloudWatch cloudWatchClient, ExecutorService execService) { + this(recordProcessorFactory, config, kinesisClient, dynamoDBClient, getMetricsFactory(cloudWatchClient, config), + execService); + } + + // Backwards compatible constructors + /** + * This constructor is for binary compatibility with code compiled against version of the KCL that only have + * constructors taking "Client" objects. + * + * @deprecated The access to this constructor will be changed in a future release. The recommended way to create + * a Worker is to use {@link Builder} + * + * @param recordProcessorFactory + * Used to get record processor instances for processing data from shards + * @param config + * Kinesis Client Library configuration + * @param kinesisClient + * Kinesis Client used for fetching data + * @param dynamoDBClient + * DynamoDB client used for checkpoints and tracking leases + * @param cloudWatchClient + * CloudWatch Client for publishing metrics + */ + @Deprecated + public Worker( + com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorFactory recordProcessorFactory, + KinesisClientLibConfiguration config, AmazonKinesisClient kinesisClient, + AmazonDynamoDBClient dynamoDBClient, AmazonCloudWatchClient cloudWatchClient) { + this(recordProcessorFactory, config, (AmazonKinesis) kinesisClient, (AmazonDynamoDB) dynamoDBClient, + (AmazonCloudWatch) cloudWatchClient); } /** - * @param recordProcessorFactory Used to get record processor instances for processing data from shards - * @param config Kinesis Client Library configuration - * @param kinesisClient Kinesis Client used for fetching data - * @param dynamoDBClient DynamoDB client used for checkpoints and tracking leases - * @param metricsFactory Metrics factory used to emit metrics - * @param execService ExecutorService to use for processing records (support for multi-threaded - * consumption) + * This constructor is for binary compatibility with code compiled against version of the KCL that only have + * constructors taking "Client" objects. + * + * @deprecated The access to this constructor will be changed in a future release. The recommended way to create + * a Worker is to use {@link Builder} + * + * @param recordProcessorFactory + * Used to get record processor instances for processing data from shards + * @param config + * Kinesis Client Library configuration + * @param kinesisClient + * Kinesis Client used for fetching data + * @param dynamoDBClient + * DynamoDB client used for checkpoints and tracking leases + * @param cloudWatchClient + * CloudWatch Client for publishing metrics + * @param execService + * ExecutorService to use for processing records (support for multi-threaded consumption) */ + @Deprecated public Worker( com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorFactory recordProcessorFactory, - KinesisClientLibConfiguration config, - AmazonKinesis kinesisClient, - AmazonDynamoDB dynamoDBClient, - IMetricsFactory metricsFactory, - ExecutorService execService) { - this( - config.getApplicationName(), - new V1ToV2RecordProcessorFactoryAdapter(recordProcessorFactory), + KinesisClientLibConfiguration config, AmazonKinesisClient kinesisClient, + AmazonDynamoDBClient dynamoDBClient, AmazonCloudWatchClient cloudWatchClient, ExecutorService execService) { + this(recordProcessorFactory, config, (AmazonKinesis) kinesisClient, (AmazonDynamoDB) dynamoDBClient, + (AmazonCloudWatch) cloudWatchClient, execService); + } + + /** + * This constructor is for binary compatibility with code compiled against version of the KCL that only have + * constructors taking "Client" objects. + * + * @deprecated The access to this constructor will be changed in a future release. The recommended way to create + * a Worker is to use {@link Builder} + * + * @param recordProcessorFactory + * Used to get record processor instances for processing data from shards + * @param config + * Kinesis Client Library configuration + * @param kinesisClient + * Kinesis Client used for fetching data + * @param dynamoDBClient + * DynamoDB client used for checkpoints and tracking leases + * @param metricsFactory + * Metrics factory used to emit metrics + * @param execService + * ExecutorService to use for processing records (support for multi-threaded consumption) + */ + @Deprecated + public Worker( + com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorFactory recordProcessorFactory, + KinesisClientLibConfiguration config, AmazonKinesisClient kinesisClient, + AmazonDynamoDBClient dynamoDBClient, IMetricsFactory metricsFactory, ExecutorService execService) { + this(recordProcessorFactory, config, (AmazonKinesis) kinesisClient, (AmazonDynamoDB) dynamoDBClient, + metricsFactory, execService); + } + + /** + * @deprecated The access to this constructor will be changed in a future release. The recommended way to create + * a Worker is to use {@link Builder} + * + * @param recordProcessorFactory + * Used to get record processor instances for processing data from shards + * @param config + * Kinesis Client Library configuration + * @param kinesisClient + * Kinesis Client used for fetching data + * @param dynamoDBClient + * DynamoDB client used for checkpoints and tracking leases + * @param metricsFactory + * Metrics factory used to emit metrics + * @param execService + * ExecutorService to use for processing records (support for multi-threaded consumption) + */ + @Deprecated + public Worker( + com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorFactory recordProcessorFactory, + KinesisClientLibConfiguration config, AmazonKinesis kinesisClient, AmazonDynamoDB dynamoDBClient, + IMetricsFactory metricsFactory, ExecutorService execService) { + this(config.getApplicationName(), new V1ToV2RecordProcessorFactoryAdapter(recordProcessorFactory), + config, new StreamConfig( - new KinesisProxyFactory(config.getKinesisCredentialsProvider(), kinesisClient) - .getProxy(config.getStreamName()), + new KinesisProxy(config, kinesisClient), config.getMaxRecords(), config.getIdleTimeBetweenReadsInMillis(), config.shouldCallProcessRecordsEvenForEmptyRecordList(), config.shouldValidateSequenceNumberBeforeCheckpointing(), config.getInitialPositionInStreamExtended()), - config.getInitialPositionInStreamExtended(), - config.getParentShardPollIntervalMillis(), - config.getShardSyncIntervalMillis(), - config.shouldCleanupLeasesUponShardCompletion(), - null, + config.getInitialPositionInStreamExtended(), config.getParentShardPollIntervalMillis(), + config.getShardSyncIntervalMillis(), config.shouldCleanupLeasesUponShardCompletion(), null, new KinesisClientLibLeaseCoordinator( new KinesisClientLeaseManager(config.getTableName(), dynamoDBClient), config.getWorkerIdentifier(), @@ -239,6 +393,7 @@ public class Worker implements Runnable { config.getEpsilonMillis(), config.getMaxLeasesForWorker(), config.getMaxLeasesToStealAtOneTime(), + config.getMaxLeaseRenewalThreads(), metricsFactory) .withInitialLeaseTableReadCapacity(config.getInitialLeaseTableReadCapacity()) .withInitialLeaseTableWriteCapacity(config.getInitialLeaseTableWriteCapacity()), @@ -247,73 +402,120 @@ public class Worker implements Runnable { config.getTaskBackoffTimeMillis(), config.getFailoverTimeMillis(), config.getSkipShardSyncAtWorkerInitializationIfLeasesExist(), - config.getShardPrioritizationStrategy()); + config.getShardPrioritizationStrategy(), + config.getRetryGetRecordsInSeconds(), + config.getMaxGetRecordsThreadPool(), + DEFAULT_WORKER_STATE_CHANGE_LISTENER); // If a region name was explicitly specified, use it as the region for Amazon Kinesis and Amazon DynamoDB. if (config.getRegionName() != null) { - Region region = RegionUtils.getRegion(config.getRegionName()); - kinesisClient.setRegion(region); - LOG.debug("The region of Amazon Kinesis client has been set to " + config.getRegionName()); - dynamoDBClient.setRegion(region); - LOG.debug("The region of Amazon DynamoDB client has been set to " + config.getRegionName()); + setField(kinesisClient, "region", kinesisClient::setRegion, RegionUtils.getRegion(config.getRegionName())); + setField(dynamoDBClient, "region", dynamoDBClient::setRegion, RegionUtils.getRegion(config.getRegionName())); } // If a dynamoDB endpoint was explicitly specified, use it to set the DynamoDB endpoint. if (config.getDynamoDBEndpoint() != null) { - dynamoDBClient.setEndpoint(config.getDynamoDBEndpoint()); - LOG.debug("The endpoint of Amazon DynamoDB client has been set to " + config.getDynamoDBEndpoint()); + setField(dynamoDBClient, "endpoint", dynamoDBClient::setEndpoint, config.getDynamoDBEndpoint()); } // If a kinesis endpoint was explicitly specified, use it to set the region of kinesis. if (config.getKinesisEndpoint() != null) { - kinesisClient.setEndpoint(config.getKinesisEndpoint()); - if (config.getRegionName() != null) { - LOG.warn("Received configuration for both region name as " + config.getRegionName() - + ", and Amazon Kinesis endpoint as " + config.getKinesisEndpoint() - + ". Amazon Kinesis endpoint will overwrite region name."); - LOG.debug("The region of Amazon Kinesis client has been overwritten to " + config.getKinesisEndpoint()); - } else { - LOG.debug("The region of Amazon Kinesis client has been set to " + config.getKinesisEndpoint()); - } + setField(kinesisClient, "endpoint", kinesisClient::setEndpoint, config.getKinesisEndpoint()); } } /** - * @param applicationName Name of the Kinesis application - * @param recordProcessorFactory Used to get record processor instances for processing data from shards - * @param streamConfig Stream configuration - * @param initialPositionInStream One of LATEST, TRIM_HORIZON, or AT_TIMESTAMP. The KinesisClientLibrary will start - * fetching data from this location in the stream when an application starts up for the first time and - * there are no checkpoints. If there are checkpoints, we start from the checkpoint position. - * @param parentShardPollIntervalMillis Wait for this long between polls to check if parent shards are done - * @param shardSyncIdleTimeMillis Time between tasks to sync leases and Kinesis shards - * @param cleanupLeasesUponShardCompletion Clean up shards we've finished processing (don't wait till they expire in - * Kinesis) - * @param checkpoint Used to get/set checkpoints - * @param leaseCoordinator Lease coordinator (coordinates currently owned leases) - * @param execService ExecutorService to use for processing records (support for multi-threaded - * consumption) - * @param metricsFactory Metrics factory used to emit metrics - * @param taskBackoffTimeMillis Backoff period when tasks encounter an exception - * @param shardPrioritization Provides prioritization logic to decide which available shards process first + * @param applicationName + * Name of the Kinesis application + * @param recordProcessorFactory + * Used to get record processor instances for processing data from shards + * @paran config + * Kinesis Library configuration + * @param streamConfig + * Stream configuration + * @param initialPositionInStream + * One of LATEST, TRIM_HORIZON, or AT_TIMESTAMP. The KinesisClientLibrary will start fetching data from + * this location in the stream when an application starts up for the first time and there are no + * checkpoints. If there are checkpoints, we start from the checkpoint position. + * @param parentShardPollIntervalMillis + * Wait for this long between polls to check if parent shards are done + * @param shardSyncIdleTimeMillis + * Time between tasks to sync leases and Kinesis shards + * @param cleanupLeasesUponShardCompletion + * Clean up shards we've finished processing (don't wait till they expire in Kinesis) + * @param checkpoint + * Used to get/set checkpoints + * @param leaseCoordinator + * Lease coordinator (coordinates currently owned leases) + * @param execService + * ExecutorService to use for processing records (support for multi-threaded consumption) + * @param metricsFactory + * Metrics factory used to emit metrics + * @param taskBackoffTimeMillis + * Backoff period when tasks encounter an exception + * @param shardPrioritization + * Provides prioritization logic to decide which available shards process first */ // NOTE: This has package level access solely for testing // CHECKSTYLE:IGNORE ParameterNumber FOR NEXT 10 LINES - Worker(String applicationName, - IRecordProcessorFactory recordProcessorFactory, - StreamConfig streamConfig, - InitialPositionInStreamExtended initialPositionInStream, - long parentShardPollIntervalMillis, - long shardSyncIdleTimeMillis, - boolean cleanupLeasesUponShardCompletion, - ICheckpoint checkpoint, - KinesisClientLibLeaseCoordinator leaseCoordinator, - ExecutorService execService, - IMetricsFactory metricsFactory, - long taskBackoffTimeMillis, - long failoverTimeMillis, - boolean skipShardSyncAtWorkerInitializationIfLeasesExist, - ShardPrioritization shardPrioritization) { + Worker(String applicationName, IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration config, + StreamConfig streamConfig, InitialPositionInStreamExtended initialPositionInStream, long parentShardPollIntervalMillis, + long shardSyncIdleTimeMillis, boolean cleanupLeasesUponShardCompletion, ICheckpoint checkpoint, + KinesisClientLibLeaseCoordinator leaseCoordinator, ExecutorService execService, + IMetricsFactory metricsFactory, long taskBackoffTimeMillis, long failoverTimeMillis, + boolean skipShardSyncAtWorkerInitializationIfLeasesExist, ShardPrioritization shardPrioritization) { + this(applicationName, recordProcessorFactory, config, streamConfig, initialPositionInStream, parentShardPollIntervalMillis, + shardSyncIdleTimeMillis, cleanupLeasesUponShardCompletion, checkpoint, leaseCoordinator, execService, + metricsFactory, taskBackoffTimeMillis, failoverTimeMillis, skipShardSyncAtWorkerInitializationIfLeasesExist, + shardPrioritization, Optional.empty(), Optional.empty(), DEFAULT_WORKER_STATE_CHANGE_LISTENER); + } + + /** + * @param applicationName + * Name of the Kinesis application + * @param recordProcessorFactory + * Used to get record processor instances for processing data from shards + * @param config + * Kinesis Library Configuration + * @param streamConfig + * Stream configuration + * @param initialPositionInStream + * One of LATEST, TRIM_HORIZON, or AT_TIMESTAMP. The KinesisClientLibrary will start fetching data from + * this location in the stream when an application starts up for the first time and there are no + * checkpoints. If there are checkpoints, we start from the checkpoint position. + * @param parentShardPollIntervalMillis + * Wait for this long between polls to check if parent shards are done + * @param shardSyncIdleTimeMillis + * Time between tasks to sync leases and Kinesis shards + * @param cleanupLeasesUponShardCompletion + * Clean up shards we've finished processing (don't wait till they expire in Kinesis) + * @param checkpoint + * Used to get/set checkpoints + * @param leaseCoordinator + * Lease coordinator (coordinates currently owned leases) + * @param execService + * ExecutorService to use for processing records (support for multi-threaded consumption) + * @param metricsFactory + * Metrics factory used to emit metrics + * @param taskBackoffTimeMillis + * Backoff period when tasks encounter an exception + * @param shardPrioritization + * Provides prioritization logic to decide which available shards process first + * @param retryGetRecordsInSeconds + * Time in seconds to wait before the worker retries to get a record. + * @param maxGetRecordsThreadPool + * Max number of threads in the getRecords thread pool. + */ + // NOTE: This has package level access solely for testing + // CHECKSTYLE:IGNORE ParameterNumber FOR NEXT 10 LINES + Worker(String applicationName, IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration config, StreamConfig streamConfig, + InitialPositionInStreamExtended initialPositionInStream, long parentShardPollIntervalMillis, + long shardSyncIdleTimeMillis, boolean cleanupLeasesUponShardCompletion, ICheckpoint checkpoint, + KinesisClientLibLeaseCoordinator leaseCoordinator, ExecutorService execService, + IMetricsFactory metricsFactory, long taskBackoffTimeMillis, long failoverTimeMillis, + boolean skipShardSyncAtWorkerInitializationIfLeasesExist, ShardPrioritization shardPrioritization, + Optional retryGetRecordsInSeconds, Optional maxGetRecordsThreadPool, WorkerStateChangeListener workerStateChangeListener) { this.applicationName = applicationName; this.recordProcessorFactory = recordProcessorFactory; + this.config = config; this.streamConfig = streamConfig; this.initialPosition = initialPositionInStream; this.parentShardPollIntervalMillis = parentShardPollIntervalMillis; @@ -323,18 +525,17 @@ public class Worker implements Runnable { this.executorService = execService; this.leaseCoordinator = leaseCoordinator; this.metricsFactory = metricsFactory; - this.controlServer = - new ShardSyncTaskManager(streamConfig.getStreamProxy(), - leaseCoordinator.getLeaseManager(), - initialPositionInStream, - cleanupLeasesUponShardCompletion, - shardSyncIdleTimeMillis, - metricsFactory, - executorService); + this.controlServer = new ShardSyncTaskManager(streamConfig.getStreamProxy(), leaseCoordinator.getLeaseManager(), + initialPositionInStream, cleanupLeasesUponShardCompletion, config.shouldIgnoreUnexpectedChildShards(), + shardSyncIdleTimeMillis, metricsFactory, executorService); this.taskBackoffTimeMillis = taskBackoffTimeMillis; - this.failoverTimeMillis = failoverTimeMillis; + this.failoverTimeMillis = failoverTimeMillis; this.skipShardSyncAtWorkerInitializationIfLeasesExist = skipShardSyncAtWorkerInitializationIfLeasesExist; this.shardPrioritization = shardPrioritization; + this.retryGetRecordsInSeconds = retryGetRecordsInSeconds; + this.maxGetRecordsThreadPool = maxGetRecordsThreadPool; + this.workerStateChangeListener = workerStateChangeListener; + workerStateChangeListener.onWorkerStateChange(WorkerStateChangeListener.WorkerState.CREATED); } /** @@ -345,8 +546,14 @@ public class Worker implements Runnable { } /** - * Start consuming data from the stream, and pass it to the application - * record processors. + * @return the leaseCoordinator + */ + KinesisClientLibLeaseCoordinator getLeaseCoordinator(){ + return leaseCoordinator; + } + + /** + * Start consuming data from the stream, and pass it to the application record processors. */ public void run() { if (shutdown) { @@ -371,45 +578,42 @@ public class Worker implements Runnable { @VisibleForTesting void runProcessLoop() { - for (int i = 0; ; i++) { - try { - boolean foundCompletedShard = false; - Set assignedShards = new HashSet<>(); - for (ShardInfo shardInfo : getShardInfoForAssignments()) { - ShardConsumer shardConsumer = createOrGetShardConsumer(shardInfo, recordProcessorFactory); - if (shardConsumer.isShutdown() && shardConsumer.getShutdownReason().equals(ShutdownReason.TERMINATE)) { - foundCompletedShard = true; - } else { - shardConsumer.consumeShard(); - } - assignedShards.add(shardInfo); - } - - if (foundCompletedShard) { - controlServer.syncShardAndLeaseInfo(null); - } - - // clean up shard consumers for unassigned shards - cleanupShardConsumers(assignedShards); - - wlog.info("Sleeping ..."); - Thread.sleep(idleTimeInMilliseconds); - } catch (Exception e) { - if (i > MAX_RETRIES) throw new RuntimeException("Giving up after " + i + " retries", e); - - LOG.error(String.format("Worker.run caught exception, sleeping for %s milli seconds!", - String.valueOf(idleTimeInMilliseconds)), e); - try { - Thread.sleep(idleTimeInMilliseconds); - } catch (InterruptedException ex) { - LOG.info("Worker: sleep interrupted after catching exception ", ex); + try { + boolean foundCompletedShard = false; + Set assignedShards = new HashSet<>(); + for (ShardInfo shardInfo : getShardInfoForAssignments()) { + ShardConsumer shardConsumer = createOrGetShardConsumer(shardInfo, recordProcessorFactory); + if (shardConsumer.isShutdown() && shardConsumer.getShutdownReason().equals(ShutdownReason.TERMINATE)) { + foundCompletedShard = true; + } else { + shardConsumer.consumeShard(); } + assignedShards.add(shardInfo); + } + + if (foundCompletedShard) { + controlServer.syncShardAndLeaseInfo(null); + } + + // clean up shard consumers for unassigned shards + cleanupShardConsumers(assignedShards); + + wlog.info("Sleeping ..."); + Thread.sleep(idleTimeInMilliseconds); + } catch (Exception e) { + LOG.error(String.format("Worker.run caught exception, sleeping for %s milli seconds!", + String.valueOf(idleTimeInMilliseconds)), e); + try { + Thread.sleep(idleTimeInMilliseconds); + } catch (InterruptedException ex) { + LOG.info("Worker: sleep interrupted after catching exception ", ex); } - wlog.resetInfoLogging(); } + wlog.resetInfoLogging(); } private void initialize() { + workerStateChangeListener.onWorkerStateChange(WorkerStateChangeListener.WorkerState.INITIALIZING); boolean isDone = false; Exception lastException = null; @@ -423,12 +627,9 @@ public class Worker implements Runnable { if (!skipShardSyncAtWorkerInitializationIfLeasesExist || leaseCoordinator.getLeaseManager().isLeaseTableEmpty()) { LOG.info("Syncing Kinesis shard info"); - ShardSyncTask shardSyncTask = - new ShardSyncTask(streamConfig.getStreamProxy(), - leaseCoordinator.getLeaseManager(), - initialPosition, - cleanupLeasesUponShardCompletion, - 0L); + ShardSyncTask shardSyncTask = new ShardSyncTask(streamConfig.getStreamProxy(), + leaseCoordinator.getLeaseManager(), initialPosition, cleanupLeasesUponShardCompletion, + config.shouldIgnoreUnexpectedChildShards(), 0L); result = new MetricsCollectingTaskDecorator(shardSyncTask, metricsFactory).call(); } else { LOG.info("Skipping shard sync per config setting (and lease table is not empty)"); @@ -462,17 +663,17 @@ public class Worker implements Runnable { if (!isDone) { throw new RuntimeException(lastException); } + workerStateChangeListener.onWorkerStateChange(WorkerStateChangeListener.WorkerState.STARTED); } /** - * NOTE: This method is internal/private to the Worker class. It has package - * access solely for testing. + * NOTE: This method is internal/private to the Worker class. It has package access solely for testing. * * This method relies on ShardInfo.equals() method returning true for ShardInfo objects which may have been * instantiated with parentShardIds in a different order (and rest of the fields being the equal). For example - * shardInfo1.equals(shardInfo2) should return true with shardInfo1 and shardInfo2 defined as follows. - * ShardInfo shardInfo1 = new ShardInfo(shardId1, concurrencyToken1, Arrays.asList("parent1", "parent2")); - * ShardInfo shardInfo2 = new ShardInfo(shardId1, concurrencyToken1, Arrays.asList("parent2", "parent1")); + * shardInfo1.equals(shardInfo2) should return true with shardInfo1 and shardInfo2 defined as follows. ShardInfo + * shardInfo1 = new ShardInfo(shardId1, concurrencyToken1, Arrays.asList("parent1", "parent2")); ShardInfo + * shardInfo2 = new ShardInfo(shardId1, concurrencyToken1, Arrays.asList("parent2", "parent1")); */ void cleanupShardConsumers(Set assignedShards) { for (ShardInfo shard : shardInfoShardConsumerMap.keySet()) { @@ -512,8 +713,57 @@ public class Worker implements Runnable { } /** - * Requests shutdown of the worker, notifying record processors, that implement {@link IShutdownNotificationAware}, - * of the impending shutdown. This gives the record processor a final chance to checkpoint. + * Starts the requestedShutdown process, and returns a future that can be used to track the process. + * + * This is deprecated in favor of {@link #startGracefulShutdown()}, which returns a more complete future, and + * indicates the process behavior + * + * @return a future that will be set once shutdown is completed. + */ + @Deprecated + public Future requestShutdown() { + + Future requestedShutdownFuture = startGracefulShutdown(); + + return new Future() { + + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + return requestedShutdownFuture.cancel(mayInterruptIfRunning); + } + + @Override + public boolean isCancelled() { + return requestedShutdownFuture.isCancelled(); + } + + @Override + public boolean isDone() { + return requestedShutdownFuture.isDone(); + } + + @Override + public Void get() throws InterruptedException, ExecutionException { + requestedShutdownFuture.get(); + return null; + } + + @Override + public Void get(long timeout, TimeUnit unit) + throws InterruptedException, ExecutionException, TimeoutException { + requestedShutdownFuture.get(timeout, unit); + return null; + } + }; + } + + /** + * Requests a graceful shutdown of the worker, notifying record processors, that implement + * {@link IShutdownNotificationAware}, of the impending shutdown. This gives the record processor a final chance to + * checkpoint. + * + * This will only create a single shutdown future. Additional attempts to start a graceful shutdown will return the + * previous future. * * It's possible that a record processor won't be notify before being shutdown. This can occur if the lease is * lost after requesting shutdown, but before the notification is dispatched. @@ -537,48 +787,85 @@ public class Worker implements Runnable { *
  • Once the worker shutdown is complete, the returned future is completed.
    • * * - * - * - * @return a Future that will be set once the shutdown is complete. + * @return a future that will be set once the shutdown has completed. True indicates that the graceful shutdown + * completed successfully. A false value indicates that a non-exception case caused the shutdown process to + * terminate early. */ - public Future requestShutdown() { - - // - // Stop accepting new leases. Once we do this we can be sure that - // no more leases will be acquired. - // - leaseCoordinator.stopLeaseTaker(); - - Collection leases = leaseCoordinator.getAssignments(); - if (leases == null || leases.isEmpty()) { - // - // If there are no leases notification is already completed, but we still need to shutdown the worker. - // - this.shutdown(); - return Futures.immediateFuture(null); - } - CountDownLatch shutdownCompleteLatch = new CountDownLatch(leases.size()); - CountDownLatch notificationCompleteLatch = new CountDownLatch(leases.size()); - for (KinesisClientLease lease : leases) { - ShutdownNotification shutdownNotification = new ShardConsumerShutdownNotification(leaseCoordinator, lease, - notificationCompleteLatch, shutdownCompleteLatch); - ShardInfo shardInfo = KinesisClientLibLeaseCoordinator.convertLeaseToAssignment(lease); - ShardConsumer consumer = shardInfoShardConsumerMap.get(shardInfo); - if (consumer != null) { - consumer.notifyShutdownRequested(shutdownNotification); - } else { - // - // There is a race condition between retrieving the current assignments, and creating the - // notification. If the a lease is lost in between these two points, we explicitly decrement the - // notification latches to clear the shutdown. - // - notificationCompleteLatch.countDown(); - shutdownCompleteLatch.countDown(); + public Future startGracefulShutdown() { + synchronized (this) { + if (gracefulShutdownFuture == null) { + gracefulShutdownFuture = gracefulShutdownCoordinator + .startGracefulShutdown(createGracefulShutdownCallable()); } } + return gracefulShutdownFuture; + } - return new ShutdownFuture(shutdownCompleteLatch, notificationCompleteLatch, this); + /** + * Creates a callable that will execute the graceful shutdown process. This callable can be used to execute graceful + * shutdowns in your own executor, or execute the shutdown synchronously. + * + * @return a callable that run the graceful shutdown process. This may return a callable that return true if the + * graceful shutdown has already been completed. + * @throws IllegalStateException + * thrown by the callable if another callable has already started the shutdown process. + */ + public Callable createGracefulShutdownCallable() { + if (isShutdownComplete()) { + return () -> true; + } + Callable startShutdown = createWorkerShutdownCallable(); + return gracefulShutdownCoordinator.createGracefulShutdownCallable(startShutdown); + } + public boolean hasGracefulShutdownStarted() { + return gracefuleShutdownStarted; + } + + @VisibleForTesting + Callable createWorkerShutdownCallable() { + return () -> { + synchronized (this) { + if (this.gracefuleShutdownStarted) { + throw new IllegalStateException("Requested shutdown has already been started"); + } + this.gracefuleShutdownStarted = true; + } + // + // Stop accepting new leases. Once we do this we can be sure that + // no more leases will be acquired. + // + leaseCoordinator.stopLeaseTaker(); + + Collection leases = leaseCoordinator.getAssignments(); + if (leases == null || leases.isEmpty()) { + // + // If there are no leases notification is already completed, but we still need to shutdown the worker. + // + this.shutdown(); + return GracefulShutdownContext.SHUTDOWN_ALREADY_COMPLETED; + } + CountDownLatch shutdownCompleteLatch = new CountDownLatch(leases.size()); + CountDownLatch notificationCompleteLatch = new CountDownLatch(leases.size()); + for (KinesisClientLease lease : leases) { + ShutdownNotification shutdownNotification = new ShardConsumerShutdownNotification(leaseCoordinator, + lease, notificationCompleteLatch, shutdownCompleteLatch); + ShardInfo shardInfo = KinesisClientLibLeaseCoordinator.convertLeaseToAssignment(lease); + ShardConsumer consumer = shardInfoShardConsumerMap.get(shardInfo); + if (consumer != null) { + consumer.notifyShutdownRequested(shutdownNotification); + } else { + // + // There is a race condition between retrieving the current assignments, and creating the + // notification. If the a lease is lost in between these two points, we explicitly decrement the + // notification latches to clear the shutdown. + // + notificationCompleteLatch.countDown(); + shutdownCompleteLatch.countDown(); + } + } + return new GracefulShutdownContext(shutdownCompleteLatch, notificationCompleteLatch, this); + }; } boolean isShutdownComplete() { @@ -589,6 +876,10 @@ public class Worker implements Runnable { return shardInfoShardConsumerMap; } + WorkerStateChangeListener getWorkerStateChangeListener() { + return workerStateChangeListener; + } + /** * Signals worker to shutdown. Worker will try initiating shutdown of all record processors. Note that if executor * services were passed to the worker by the user, worker will not attempt to shutdown those resources. @@ -619,11 +910,12 @@ public class Worker implements Runnable { // Lost leases will force Worker to begin shutdown process for all shard consumers in // Worker.run(). leaseCoordinator.stop(); + workerStateChangeListener.onWorkerStateChange(WorkerStateChangeListener.WorkerState.SHUT_DOWN); } /** - * Perform final shutdown related tasks for the worker including shutting down worker owned - * executor services, threads, etc. + * Perform final shutdown related tasks for the worker including shutting down worker owned executor services, + * threads, etc. */ private void finalShutdown() { LOG.info("Starting worker's final shutdown."); @@ -641,7 +933,7 @@ public class Worker implements Runnable { /** * Returns whether worker can shutdown immediately. Note that this method is called from Worker's {{@link #run()} * method before every loop run, so method must do minimum amount of work to not impact shard processing timings. - * + * * @return Whether worker should shutdown immediately. */ @VisibleForTesting @@ -664,14 +956,15 @@ public class Worker implements Runnable { } /** - * NOTE: This method is internal/private to the Worker class. It has package - * access solely for testing. + * NOTE: This method is internal/private to the Worker class. It has package access solely for testing. * - * @param shardInfo Kinesis shard info - * @param factory RecordProcessor factory + * @param shardInfo + * Kinesis shard info + * @param processorFactory + * RecordProcessor factory * @return ShardConsumer for the shard */ - ShardConsumer createOrGetShardConsumer(ShardInfo shardInfo, IRecordProcessorFactory factory) { + ShardConsumer createOrGetShardConsumer(ShardInfo shardInfo, IRecordProcessorFactory processorFactory) { ShardConsumer consumer = shardInfoShardConsumerMap.get(shardInfo); // Instantiate a new consumer if we don't have one, or the one we // had was from an earlier @@ -680,27 +973,37 @@ public class Worker implements Runnable { // completely processed (shutdown reason terminate). if ((consumer == null) || (consumer.isShutdown() && consumer.getShutdownReason().equals(ShutdownReason.ZOMBIE))) { - consumer = buildConsumer(shardInfo, factory); + consumer = buildConsumer(shardInfo, processorFactory); shardInfoShardConsumerMap.put(shardInfo, consumer); wlog.infoForce("Created new shardConsumer for : " + shardInfo); } return consumer; } - protected ShardConsumer buildConsumer(ShardInfo shardInfo, IRecordProcessorFactory factory) { - IRecordProcessor recordProcessor = factory.createProcessor(); + protected ShardConsumer buildConsumer(ShardInfo shardInfo, IRecordProcessorFactory processorFactory) { + IRecordProcessor recordProcessor = processorFactory.createProcessor(); - return new ShardConsumer(shardInfo, streamConfig, checkpointTracker, recordProcessor, - leaseCoordinator.getLeaseManager(), parentShardPollIntervalMillis, cleanupLeasesUponShardCompletion, - executorService, metricsFactory, taskBackoffTimeMillis, skipShardSyncAtWorkerInitializationIfLeasesExist); + return new ShardConsumer(shardInfo, + streamConfig, + checkpointTracker, + recordProcessor, + leaseCoordinator.getLeaseManager(), + parentShardPollIntervalMillis, + cleanupLeasesUponShardCompletion, + executorService, + metricsFactory, + taskBackoffTimeMillis, + skipShardSyncAtWorkerInitializationIfLeasesExist, + retryGetRecordsInSeconds, + maxGetRecordsThreadPool, + config); } /** - * Logger for suppressing too much INFO logging. To avoid too much logging - * information Worker will output logging at INFO level for a single pass - * through the main loop every minute. At DEBUG level it will output all - * INFO logs on every pass. + * Logger for suppressing too much INFO logging. To avoid too much logging information Worker will output logging at + * INFO level for a single pass through the main loop every minute. At DEBUG level it will output all INFO logs on + * every pass. */ private static class WorkerLog { @@ -754,107 +1057,31 @@ public class Worker implements Runnable { } } - // Backwards compatible constructors - /** - * This constructor is for binary compatibility with code compiled against - * version of the KCL that only have constructors taking "Client" objects. - * - * @param recordProcessorFactory Used to get record processor instances for processing data from shards - * @param config Kinesis Client Library configuration - * @param kinesisClient Kinesis Client used for fetching data - * @param dynamoDBClient DynamoDB client used for checkpoints and tracking leases - * @param cloudWatchClient CloudWatch Client for publishing metrics - */ - public Worker( - com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorFactory recordProcessorFactory, - KinesisClientLibConfiguration config, - AmazonKinesisClient kinesisClient, - AmazonDynamoDBClient dynamoDBClient, - AmazonCloudWatchClient cloudWatchClient) { - this(recordProcessorFactory, - config, - (AmazonKinesis) kinesisClient, - (AmazonDynamoDB) dynamoDBClient, - (AmazonCloudWatch) cloudWatchClient); - } - - /** - * This constructor is for binary compatibility with code compiled against - * version of the KCL that only have constructors taking "Client" objects. - * - * @param recordProcessorFactory Used to get record processor instances for processing data from shards - * @param config Kinesis Client Library configuration - * @param kinesisClient Kinesis Client used for fetching data - * @param dynamoDBClient DynamoDB client used for checkpoints and tracking leases - * @param cloudWatchClient CloudWatch Client for publishing metrics - * @param execService ExecutorService to use for processing records (support for multi-threaded - * consumption) - */ - public Worker( - com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorFactory recordProcessorFactory, - KinesisClientLibConfiguration config, - AmazonKinesisClient kinesisClient, - AmazonDynamoDBClient dynamoDBClient, - AmazonCloudWatchClient cloudWatchClient, - ExecutorService execService) { - this(recordProcessorFactory, - config, - (AmazonKinesis) kinesisClient, - (AmazonDynamoDB) dynamoDBClient, - (AmazonCloudWatch) cloudWatchClient, - execService); - } - - /** - * This constructor is for binary compatibility with code compiled against - * version of the KCL that only have constructors taking "Client" objects. - * - * @param recordProcessorFactory Used to get record processor instances for processing data from shards - * @param config Kinesis Client Library configuration - * @param kinesisClient Kinesis Client used for fetching data - * @param dynamoDBClient DynamoDB client used for checkpoints and tracking leases - * @param metricsFactory Metrics factory used to emit metrics - * @param execService ExecutorService to use for processing records (support for multi-threaded - * consumption) - */ - public Worker( - com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorFactory recordProcessorFactory, - KinesisClientLibConfiguration config, - AmazonKinesisClient kinesisClient, - AmazonDynamoDBClient dynamoDBClient, - IMetricsFactory metricsFactory, - ExecutorService execService) { - this(recordProcessorFactory, - config, - (AmazonKinesis) kinesisClient, - (AmazonDynamoDB) dynamoDBClient, - metricsFactory, - execService); + @VisibleForTesting + StreamConfig getStreamConfig() { + return streamConfig; } /** * Given configuration, returns appropriate metrics factory. - * @param cloudWatchClient Amazon CloudWatch client - * @param config KinesisClientLibConfiguration + * + * @param cloudWatchClient + * Amazon CloudWatch client + * @param config + * KinesisClientLibConfiguration * @return Returns metrics factory based on the config. */ - private static IMetricsFactory getMetricsFactory( - AmazonCloudWatch cloudWatchClient, KinesisClientLibConfiguration config) { + private static IMetricsFactory getMetricsFactory(AmazonCloudWatch cloudWatchClient, + KinesisClientLibConfiguration config) { IMetricsFactory metricsFactory; if (config.getMetricsLevel() == MetricsLevel.NONE) { metricsFactory = new NullMetricsFactory(); } else { if (config.getRegionName() != null) { - Region region = RegionUtils.getRegion(config.getRegionName()); - cloudWatchClient.setRegion(region); - LOG.debug("The region of Amazon CloudWatch client has been set to " + config.getRegionName()); + setField(cloudWatchClient, "region", cloudWatchClient::setRegion, RegionUtils.getRegion(config.getRegionName())); } - metricsFactory = new WorkerCWMetricsFactory( - cloudWatchClient, - config.getApplicationName(), - config.getMetricsBufferTimeMillis(), - config.getMetricsMaxQueueSize(), - config.getMetricsLevel(), + metricsFactory = new WorkerCWMetricsFactory(cloudWatchClient, config.getApplicationName(), + config.getMetricsBufferTimeMillis(), config.getMetricsMaxQueueSize(), config.getMetricsLevel(), config.getMetricsEnabledDimensions()); } return metricsFactory; @@ -862,6 +1089,7 @@ public class Worker implements Runnable { /** * Returns default executor service that should be used by the worker. + * * @return Default executor service that should be used by the worker. */ private static ExecutorService getExecutorService() { @@ -869,27 +1097,29 @@ public class Worker implements Runnable { return new WorkerThreadPoolExecutor(threadFactory); } - /** - * Extension to CWMetricsFactory, so worker can identify whether it owns the metrics factory instance - * or not. - * Visible and non-final only for testing. - */ - static class WorkerCWMetricsFactory extends CWMetricsFactory { - - WorkerCWMetricsFactory(AmazonCloudWatch cloudWatchClient, - String namespace, - long bufferTimeMillis, - int maxQueueSize, - MetricsLevel metricsLevel, - Set metricsEnabledDimensions) { - super(cloudWatchClient, namespace, bufferTimeMillis, - maxQueueSize, metricsLevel, metricsEnabledDimensions); + private static void setField(final S source, final String field, final Consumer t, T value) { + try { + t.accept(value); + } catch (UnsupportedOperationException e) { + LOG.debug("Exception thrown while trying to set " + field + ", indicating that " + + source.getClass().getSimpleName() + "is immutable.", e); } } /** - * Extension to ThreadPoolExecutor, so worker can identify whether it owns the executor service instance - * or not. + * Extension to CWMetricsFactory, so worker can identify whether it owns the metrics factory instance or not. + * Visible and non-final only for testing. + */ + static class WorkerCWMetricsFactory extends CWMetricsFactory { + + WorkerCWMetricsFactory(AmazonCloudWatch cloudWatchClient, String namespace, long bufferTimeMillis, + int maxQueueSize, MetricsLevel metricsLevel, Set metricsEnabledDimensions) { + super(cloudWatchClient, namespace, bufferTimeMillis, maxQueueSize, metricsLevel, metricsEnabledDimensions); + } + } + + /** + * Extension to ThreadPoolExecutor, so worker can identify whether it owns the executor service instance or not. * Visible and non-final only for testing. */ static class WorkerThreadPoolExecutor extends ThreadPoolExecutor { @@ -908,39 +1138,62 @@ public class Worker implements Runnable { public static class Builder { private IRecordProcessorFactory recordProcessorFactory; + @Setter @Accessors(fluent = true) private KinesisClientLibConfiguration config; + @Setter @Accessors(fluent = true) private AmazonKinesis kinesisClient; + @Setter @Accessors(fluent = true) private AmazonDynamoDB dynamoDBClient; + @Setter @Accessors(fluent = true) private AmazonCloudWatch cloudWatchClient; + @Setter @Accessors(fluent = true) private IMetricsFactory metricsFactory; + @Setter @Accessors(fluent = true) + private ILeaseManager leaseManager; + @Setter @Accessors(fluent = true) private ExecutorService execService; + @Setter @Accessors(fluent = true) private ShardPrioritization shardPrioritization; + @Setter @Accessors(fluent = true) + private IKinesisProxy kinesisProxy; + @Setter @Accessors(fluent = true) + private WorkerStateChangeListener workerStateChangeListener; - /** - * Default constructor. - */ - public Builder() { + @VisibleForTesting + AmazonKinesis getKinesisClient() { + return kinesisClient; + } + + @VisibleForTesting + AmazonDynamoDB getDynamoDBClient() { + return dynamoDBClient; + } + + @VisibleForTesting + AmazonCloudWatch getCloudWatchClient() { + return cloudWatchClient; } /** - * Provide a V1 - * {@link com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessor IRecordProcessor}. + * Provide a V1 {@link com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessor + * IRecordProcessor}. * - * @param recordProcessorFactory Used to get record processor instances for processing data from shards + * @param recordProcessorFactory + * Used to get record processor instances for processing data from shards * @return A reference to this updated object so that method calls can be chained together. */ public Builder recordProcessorFactory( - com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorFactory - recordProcessorFactory) { + com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorFactory recordProcessorFactory) { this.recordProcessorFactory = new V1ToV2RecordProcessorFactoryAdapter(recordProcessorFactory); return this; } /** - * Provide a V2 - * {@link com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessor IRecordProcessor}. + * Provide a V2 {@link com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessor + * IRecordProcessor}. * - * @param recordProcessorFactory Used to get record processor instances for processing data from shards + * @param recordProcessorFactory + * Used to get record processor instances for processing data from shards * @return A reference to this updated object so that method calls can be chained together. */ public Builder recordProcessorFactory(IRecordProcessorFactory recordProcessorFactory) { @@ -948,85 +1201,6 @@ public class Worker implements Runnable { return this; } - /** - * Set the Worker config. - * - * @param config Kinesis Client Library configuration - * @return A reference to this updated object so that method calls can be chained together. - */ - public Builder config(KinesisClientLibConfiguration config) { - this.config = config; - return this; - } - - /** - * Set the Kinesis client. - * - * @param kinesisClient Kinesis Client used for fetching data - * @return A reference to this updated object so that method calls can be chained together. - */ - public Builder kinesisClient(AmazonKinesis kinesisClient) { - this.kinesisClient = kinesisClient; - return this; - } - - /** - * Set the DynamoDB client. - * - * @param dynamoDBClient DynamoDB client used for checkpoints and tracking leases - * @return A reference to this updated object so that method calls can be chained together. - */ - public Builder dynamoDBClient(AmazonDynamoDB dynamoDBClient) { - this.dynamoDBClient = dynamoDBClient; - return this; - } - - /** - * Set the Cloudwatch client. - * - * @param cloudWatchClient CloudWatch Client for publishing metrics - * @return A reference to this updated object so that method calls can be chained together. - */ - public Builder cloudWatchClient(AmazonCloudWatch cloudWatchClient) { - this.cloudWatchClient = cloudWatchClient; - return this; - } - - /** - * Set the metrics factory. - * - * @param metricsFactory Metrics factory used to emit metrics - * @return A reference to this updated object so that method calls can be chained together. - */ - public Builder metricsFactory(IMetricsFactory metricsFactory) { - this.metricsFactory = metricsFactory; - return this; - } - - /** - * Set the executor service for processing records. - * - * @param execService ExecutorService to use for processing records (support for multi-threaded consumption) - * @return A reference to this updated object so that method calls can be chained together. - */ - public Builder execService(ExecutorService execService) { - this.execService = execService; - return this; - } - - /** - * Provides logic how to prioritize shard processing. - * - * @param shardPrioritization - * shardPrioritization is responsible to order shards before processing - * - * @return A reference to this updated object so that method calls can be chained together. - */ - public Builder shardPrioritization(ShardPrioritization shardPrioritization) { - this.shardPrioritization = shardPrioritization; - return this; - } - /** * Build the Worker instance. * @@ -1040,64 +1214,68 @@ public class Worker implements Runnable { "Kinesis Client Library configuration needs to be provided to build Worker"); } if (recordProcessorFactory == null) { - throw new IllegalArgumentException( - "A Record Processor Factory needs to be provided to build Worker"); + throw new IllegalArgumentException("A Record Processor Factory needs to be provided to build Worker"); } if (execService == null) { execService = getExecutorService(); } if (kinesisClient == null) { - kinesisClient = new AmazonKinesisClient(config.getKinesisCredentialsProvider(), - config.getKinesisClientConfiguration()); + kinesisClient = createClient(AmazonKinesisClientBuilder.standard(), + config.getKinesisCredentialsProvider(), + config.getKinesisClientConfiguration(), + config.getKinesisEndpoint(), + config.getRegionName()); } if (dynamoDBClient == null) { - dynamoDBClient = new AmazonDynamoDBClient(config.getDynamoDBCredentialsProvider(), - config.getDynamoDBClientConfiguration()); + dynamoDBClient = createClient(AmazonDynamoDBClientBuilder.standard(), + config.getDynamoDBCredentialsProvider(), + config.getDynamoDBClientConfiguration(), + config.getDynamoDBEndpoint(), + config.getRegionName()); } if (cloudWatchClient == null) { - cloudWatchClient = new AmazonCloudWatchClient(config.getCloudWatchCredentialsProvider(), - config.getCloudWatchClientConfiguration()); + cloudWatchClient = createClient(AmazonCloudWatchClientBuilder.standard(), + config.getCloudWatchCredentialsProvider(), + config.getCloudWatchClientConfiguration(), + null, + config.getRegionName()); } // If a region name was explicitly specified, use it as the region for Amazon Kinesis and Amazon DynamoDB. if (config.getRegionName() != null) { - Region region = RegionUtils.getRegion(config.getRegionName()); - cloudWatchClient.setRegion(region); - LOG.debug("The region of Amazon CloudWatch client has been set to " + config.getRegionName()); - kinesisClient.setRegion(region); - LOG.debug("The region of Amazon Kinesis client has been set to " + config.getRegionName()); - dynamoDBClient.setRegion(region); - LOG.debug("The region of Amazon DynamoDB client has been set to " + config.getRegionName()); + setField(cloudWatchClient, "region", cloudWatchClient::setRegion, RegionUtils.getRegion(config.getRegionName())); + setField(kinesisClient, "region", kinesisClient::setRegion, RegionUtils.getRegion(config.getRegionName())); + setField(dynamoDBClient, "region", dynamoDBClient::setRegion, RegionUtils.getRegion(config.getRegionName())); } // If a dynamoDB endpoint was explicitly specified, use it to set the DynamoDB endpoint. if (config.getDynamoDBEndpoint() != null) { - dynamoDBClient.setEndpoint(config.getDynamoDBEndpoint()); - LOG.debug("The endpoint of Amazon DynamoDB client has been set to " + config.getDynamoDBEndpoint()); + setField(dynamoDBClient, "endpoint", dynamoDBClient::setEndpoint, config.getDynamoDBEndpoint()); } // If a kinesis endpoint was explicitly specified, use it to set the region of kinesis. if (config.getKinesisEndpoint() != null) { - kinesisClient.setEndpoint(config.getKinesisEndpoint()); - if (config.getRegionName() != null) { - LOG.warn("Received configuration for both region name as " + config.getRegionName() - + ", and Amazon Kinesis endpoint as " + config.getKinesisEndpoint() - + ". Amazon Kinesis endpoint will overwrite region name."); - LOG.debug("The region of Amazon Kinesis client has been overwritten to " - + config.getKinesisEndpoint()); - } else { - LOG.debug("The region of Amazon Kinesis client has been set to " + config.getKinesisEndpoint()); - } + setField(kinesisClient, "endpoint", kinesisClient::setEndpoint, config.getKinesisEndpoint()); } if (metricsFactory == null) { metricsFactory = getMetricsFactory(cloudWatchClient, config); } + if (leaseManager == null) { + leaseManager = new KinesisClientLeaseManager(config.getTableName(), dynamoDBClient); + } if (shardPrioritization == null) { shardPrioritization = new ParentsFirstShardPrioritization(1); } + if (kinesisProxy == null) { + kinesisProxy = new KinesisProxy(config, kinesisClient); + } + + if (workerStateChangeListener == null) { + workerStateChangeListener = DEFAULT_WORKER_STATE_CHANGE_LISTENER; + } return new Worker(config.getApplicationName(), recordProcessorFactory, - new StreamConfig(new KinesisProxyFactory(config.getKinesisCredentialsProvider(), - kinesisClient).getProxy(config.getStreamName()), + config, + new StreamConfig(kinesisProxy, config.getMaxRecords(), config.getIdleTimeBetweenReadsInMillis(), config.shouldCallProcessRecordsEvenForEmptyRecordList(), @@ -1108,13 +1286,13 @@ public class Worker implements Runnable { config.getShardSyncIntervalMillis(), config.shouldCleanupLeasesUponShardCompletion(), null, - new KinesisClientLibLeaseCoordinator(new KinesisClientLeaseManager(config.getTableName(), - dynamoDBClient), + new KinesisClientLibLeaseCoordinator(leaseManager, config.getWorkerIdentifier(), config.getFailoverTimeMillis(), config.getEpsilonMillis(), config.getMaxLeasesForWorker(), config.getMaxLeasesToStealAtOneTime(), + config.getMaxLeaseRenewalThreads(), metricsFactory) .withInitialLeaseTableReadCapacity(config.getInitialLeaseTableReadCapacity()) .withInitialLeaseTableWriteCapacity(config.getInitialLeaseTableWriteCapacity()), @@ -1123,9 +1301,35 @@ public class Worker implements Runnable { config.getTaskBackoffTimeMillis(), config.getFailoverTimeMillis(), config.getSkipShardSyncAtWorkerInitializationIfLeasesExist(), - shardPrioritization); - + shardPrioritization, + config.getRetryGetRecordsInSeconds(), + config.getMaxGetRecordsThreadPool(), + workerStateChangeListener); } + > R createClient(final T builder, + final AWSCredentialsProvider credentialsProvider, + final ClientConfiguration clientConfiguration, + final String endpointUrl, + final String region) { + if (credentialsProvider != null) { + builder.withCredentials(credentialsProvider); + } + if (clientConfiguration != null) { + builder.withClientConfiguration(clientConfiguration); + } + if (StringUtils.isNotEmpty(endpointUrl)) { + LOG.warn("Received configuration for endpoint as " + endpointUrl + ", and region as " + + region + "."); + builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpointUrl, region)); + } else if (StringUtils.isNotEmpty(region)) { + LOG.warn("Received configuration for region as " + region + "."); + builder.withRegion(region); + } else { + LOG.warn("No configuration received for endpoint and region, will default region to us-east-1"); + builder.withRegion(Regions.US_EAST_1); + } + return builder.build(); + } } } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/WorkerStateChangeListener.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/WorkerStateChangeListener.java new file mode 100644 index 00000000..36ee39f0 --- /dev/null +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/WorkerStateChangeListener.java @@ -0,0 +1,16 @@ +package com.amazonaws.services.kinesis.clientlibrary.lib.worker; + +/** + * A listener for callbacks on changes worker state + */ +@FunctionalInterface +public interface WorkerStateChangeListener { + enum WorkerState { + CREATED, + INITIALIZING, + STARTED, + SHUT_DOWN + } + + void onWorkerStateChange(WorkerState newState); +} diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/IKinesisProxy.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/IKinesisProxy.java index df7f951d..73a868dd 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/IKinesisProxy.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/IKinesisProxy.java @@ -48,10 +48,17 @@ public interface IKinesisProxy { /** * Fetch information about stream. Useful for fetching the list of shards in a stream. * + * @deprecated Going forward this method is + * being deprecated. This method uses DescribeStream call, which is throttled at 10 calls per account by default. + * If possible try to use ListShards call available in the client, or use the getShardList or getAllShards to get + * shard info. To make DescribeStream calls, use the AmazonKinesis client directly instead of using KinesisProxy. + * This method will be removed in the next major/minor release. + * * @param startShardId exclusive start shardId - used when paginating the list of shards. * @return DescribeStreamOutput object containing a description of the stream. * @throws ResourceNotFoundException The Kinesis stream was not found */ + @Deprecated DescribeStreamResult getStreamInfo(String startShardId) throws ResourceNotFoundException; /** diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/IKinesisProxyFactory.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/IKinesisProxyFactory.java index 0467b8e4..dd3c82e5 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/IKinesisProxyFactory.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/IKinesisProxyFactory.java @@ -17,7 +17,11 @@ package com.amazonaws.services.kinesis.clientlibrary.proxies; /** * Interface for a KinesisProxyFactory. * + * @deprecated Deprecating since KinesisProxy is just created once, there is no use of a factory. There is no + * replacement for this class. This class will be removed in the next major/minor release. + * */ +@Deprecated public interface IKinesisProxyFactory { /** diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxy.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxy.java index de330dc9..fe830444 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxy.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxy.java @@ -1,34 +1,42 @@ /* - * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.clientlibrary.proxies; import java.nio.ByteBuffer; +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Date; import java.util.EnumSet; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; -import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.services.kinesis.AmazonKinesis; import com.amazonaws.services.kinesis.AmazonKinesisClient; +import com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisClientLibConfiguration; import com.amazonaws.services.kinesis.model.DescribeStreamRequest; import com.amazonaws.services.kinesis.model.DescribeStreamResult; import com.amazonaws.services.kinesis.model.ExpiredIteratorException; @@ -38,13 +46,21 @@ import com.amazonaws.services.kinesis.model.GetShardIteratorRequest; import com.amazonaws.services.kinesis.model.GetShardIteratorResult; import com.amazonaws.services.kinesis.model.InvalidArgumentException; import com.amazonaws.services.kinesis.model.LimitExceededException; +import com.amazonaws.services.kinesis.model.ListShardsRequest; +import com.amazonaws.services.kinesis.model.ListShardsResult; import com.amazonaws.services.kinesis.model.PutRecordRequest; import com.amazonaws.services.kinesis.model.PutRecordResult; +import com.amazonaws.services.kinesis.model.ResourceInUseException; import com.amazonaws.services.kinesis.model.ResourceNotFoundException; import com.amazonaws.services.kinesis.model.Shard; import com.amazonaws.services.kinesis.model.ShardIteratorType; import com.amazonaws.services.kinesis.model.StreamStatus; +import lombok.AccessLevel; +import lombok.Data; +import lombok.Getter; +import lombok.Setter; + /** * Kinesis proxy - used to make calls to Amazon Kinesis (e.g. fetch data records and list of shards). */ @@ -54,13 +70,26 @@ public class KinesisProxy implements IKinesisProxyExtended { private static final EnumSet EXPECTED_ITERATOR_TYPES = EnumSet .of(ShardIteratorType.AT_SEQUENCE_NUMBER, ShardIteratorType.AFTER_SEQUENCE_NUMBER); + public static final int MAX_CACHE_MISSES_BEFORE_RELOAD = 1000; + public static final Duration CACHE_MAX_ALLOWED_AGE = Duration.of(30, ChronoUnit.SECONDS); + public static final int CACHE_MISS_WARNING_MODULUS = 250; private static String defaultServiceName = "kinesis"; private static String defaultRegionId = "us-east-1";; private AmazonKinesis client; private AWSCredentialsProvider credentialsProvider; - private AtomicReference> listOfShardsSinceLastGet = new AtomicReference<>(); + + private ShardIterationState shardIterationState = null; + + @Setter(AccessLevel.PACKAGE) + private volatile Map cachedShardMap = null; + @Setter(AccessLevel.PACKAGE) + @Getter(AccessLevel.PACKAGE) + private volatile Instant lastCacheUpdateTime = null; + @Setter(AccessLevel.PACKAGE) + @Getter(AccessLevel.PACKAGE) + private AtomicInteger cacheMisses = new AtomicInteger(0); private final String streamName; @@ -68,51 +97,24 @@ public class KinesisProxy implements IKinesisProxyExtended { private static final int DEFAULT_DESCRIBE_STREAM_RETRY_TIMES = 50; private final long describeStreamBackoffTimeInMillis; private final int maxDescribeStreamRetryAttempts; + private final long listShardsBackoffTimeInMillis; + private final int maxListShardsRetryAttempts; + private boolean isKinesisClient = true; /** - * Public constructor. + * @deprecated We expect the client to be passed to the proxy, and the proxy will not require to create it. * - * @param streamName Data records will be fetched from this stream - * @param credentialProvider Provides credentials for signing Kinesis requests - * @param endpoint Kinesis endpoint + * @param credentialProvider + * @param endpoint + * @param serviceName + * @param regionId + * @return */ - - public KinesisProxy(final String streamName, AWSCredentialsProvider credentialProvider, String endpoint) { - this(streamName, credentialProvider, endpoint, defaultServiceName, defaultRegionId, - DEFAULT_DESCRIBE_STREAM_BACKOFF_MILLIS, DEFAULT_DESCRIBE_STREAM_RETRY_TIMES); - } - - /** - * Public constructor. - * - * @param streamName Data records will be fetched from this stream - * @param credentialProvider Provides credentials for signing Kinesis requests - * @param endpoint Kinesis endpoint - * @param serviceName service name - * @param regionId region id - * @param describeStreamBackoffTimeInMillis Backoff time for DescribeStream calls in milliseconds - * @param maxDescribeStreamRetryAttempts Number of retry attempts for DescribeStream calls - */ - public KinesisProxy(final String streamName, - AWSCredentialsProvider credentialProvider, - String endpoint, - String serviceName, - String regionId, - long describeStreamBackoffTimeInMillis, - int maxDescribeStreamRetryAttempts) { - this(streamName, credentialProvider, buildClientSettingEndpoint(credentialProvider, - endpoint, - serviceName, - regionId), describeStreamBackoffTimeInMillis, maxDescribeStreamRetryAttempts); - - - LOG.debug("KinesisProxy has created a kinesisClient"); - } - + @Deprecated private static AmazonKinesisClient buildClientSettingEndpoint(AWSCredentialsProvider credentialProvider, - String endpoint, - String serviceName, - String regionId) { + String endpoint, + String serviceName, + String regionId) { AmazonKinesisClient client = new AmazonKinesisClient(credentialProvider); client.setEndpoint(endpoint); client.setSignerRegionOverride(regionId); @@ -122,24 +124,123 @@ public class KinesisProxy implements IKinesisProxyExtended { /** * Public constructor. * + * @deprecated Deprecating constructor, this constructor doesn't use AWS best practices, moving forward please use + * {@link #KinesisProxy(KinesisClientLibConfiguration, AmazonKinesis)} or + * {@link #KinesisProxy(String, AmazonKinesis, long, int, long, int)} to create the object. Will be removed in the + * next major/minor release. + * + * @param streamName Data records will be fetched from this stream + * @param credentialProvider Provides credentials for signing Kinesis requests + * @param endpoint Kinesis endpoint + */ + @Deprecated + public KinesisProxy(final String streamName, AWSCredentialsProvider credentialProvider, String endpoint) { + this(streamName, credentialProvider, endpoint, defaultServiceName, defaultRegionId, + DEFAULT_DESCRIBE_STREAM_BACKOFF_MILLIS, DEFAULT_DESCRIBE_STREAM_RETRY_TIMES, + KinesisClientLibConfiguration.DEFAULT_LIST_SHARDS_BACKOFF_TIME_IN_MILLIS, + KinesisClientLibConfiguration.DEFAULT_MAX_LIST_SHARDS_RETRY_ATTEMPTS); + } + + /** + * Public constructor. + * + * @deprecated Deprecating constructor, this constructor doesn't use AWS best practices, moving forward please use + * {@link #KinesisProxy(KinesisClientLibConfiguration, AmazonKinesis)} or + * {@link #KinesisProxy(String, AmazonKinesis, long, int, long, int)} to create the object. Will be removed in the + * next major/minor release. + * + * @param streamName Data records will be fetched from this stream + * @param credentialProvider Provides credentials for signing Kinesis requests + * @param endpoint Kinesis endpoint + * @param serviceName service name + * @param regionId region id + * @param describeStreamBackoffTimeInMillis Backoff time for DescribeStream calls in milliseconds + * @param maxDescribeStreamRetryAttempts Number of retry attempts for DescribeStream calls + */ + @Deprecated + public KinesisProxy(final String streamName, + AWSCredentialsProvider credentialProvider, + String endpoint, + String serviceName, + String regionId, + long describeStreamBackoffTimeInMillis, + int maxDescribeStreamRetryAttempts, + long listShardsBackoffTimeInMillis, + int maxListShardsRetryAttempts) { + this(streamName, + credentialProvider, + buildClientSettingEndpoint(credentialProvider, endpoint, serviceName, regionId), + describeStreamBackoffTimeInMillis, + maxDescribeStreamRetryAttempts, + listShardsBackoffTimeInMillis, + maxListShardsRetryAttempts); + LOG.debug("KinesisProxy has created a kinesisClient"); + } + + /** + * Public constructor. + * + * @deprecated Deprecating constructor, this constructor doesn't use AWS best practices, moving forward please use + * {@link #KinesisProxy(KinesisClientLibConfiguration, AmazonKinesis)} or + * {@link #KinesisProxy(String, AmazonKinesis, long, int, long, int)} to create the object. Will be removed in the + * next major/minor release. + * * @param streamName Data records will be fetched from this stream * @param credentialProvider Provides credentials for signing Kinesis requests * @param kinesisClient Kinesis client (used to fetch data from Kinesis) * @param describeStreamBackoffTimeInMillis Backoff time for DescribeStream calls in milliseconds * @param maxDescribeStreamRetryAttempts Number of retry attempts for DescribeStream calls */ + @Deprecated public KinesisProxy(final String streamName, AWSCredentialsProvider credentialProvider, AmazonKinesis kinesisClient, long describeStreamBackoffTimeInMillis, - int maxDescribeStreamRetryAttempts) { - this.streamName = streamName; + int maxDescribeStreamRetryAttempts, + long listShardsBackoffTimeInMillis, + int maxListShardsRetryAttempts) { + this(streamName, kinesisClient, describeStreamBackoffTimeInMillis, maxDescribeStreamRetryAttempts, + listShardsBackoffTimeInMillis, maxListShardsRetryAttempts); this.credentialsProvider = credentialProvider; + LOG.debug("KinesisProxy( " + streamName + ")"); + } + + /** + * Public constructor. + * @param config + */ + public KinesisProxy(final KinesisClientLibConfiguration config, final AmazonKinesis client) { + this(config.getStreamName(), + client, + DEFAULT_DESCRIBE_STREAM_BACKOFF_MILLIS, + DEFAULT_DESCRIBE_STREAM_RETRY_TIMES, + config.getListShardsBackoffTimeInMillis(), + config.getMaxListShardsRetryAttempts()); + this.credentialsProvider = config.getKinesisCredentialsProvider(); + } + + public KinesisProxy(final String streamName, + final AmazonKinesis client, + final long describeStreamBackoffTimeInMillis, + final int maxDescribeStreamRetryAttempts, + final long listShardsBackoffTimeInMillis, + final int maxListShardsRetryAttempts) { + this.streamName = streamName; + this.client = client; this.describeStreamBackoffTimeInMillis = describeStreamBackoffTimeInMillis; this.maxDescribeStreamRetryAttempts = maxDescribeStreamRetryAttempts; - this.client = kinesisClient; + this.listShardsBackoffTimeInMillis = listShardsBackoffTimeInMillis; + this.maxListShardsRetryAttempts = maxListShardsRetryAttempts; - LOG.debug("KinesisProxy( " + streamName + ")"); + try { + if (Class.forName("com.amazonaws.services.dynamodbv2.streamsadapter.AmazonDynamoDBStreamsAdapterClient") + .isAssignableFrom(client.getClass())) { + isKinesisClient = false; + LOG.debug("Client is DynamoDb client, will use DescribeStream."); + } + } catch (ClassNotFoundException e) { + LOG.debug("Client is Kinesis Client, using ListShards instead of DescribeStream."); + } } /** @@ -162,16 +263,20 @@ public class KinesisProxy implements IKinesisProxyExtended { * {@inheritDoc} */ @Override + @Deprecated public DescribeStreamResult getStreamInfo(String startShardId) - throws ResourceNotFoundException, LimitExceededException { + throws ResourceNotFoundException, LimitExceededException { final DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest(); describeStreamRequest.setRequestCredentials(credentialsProvider.getCredentials()); describeStreamRequest.setStreamName(streamName); describeStreamRequest.setExclusiveStartShardId(startShardId); DescribeStreamResult response = null; + + LimitExceededException lastException = null; + int remainingRetryTimes = this.maxDescribeStreamRetryAttempts; // Call DescribeStream, with backoff and retries (if we get LimitExceededException). - while ((remainingRetryTimes >= 0) && (response == null)) { + while (response == null) { try { response = client.describeStream(describeStreamRequest); } catch (LimitExceededException le) { @@ -182,8 +287,15 @@ public class KinesisProxy implements IKinesisProxyExtended { } catch (InterruptedException ie) { LOG.debug("Stream " + streamName + " : Sleep was interrupted ", ie); } + lastException = le; } remainingRetryTimes--; + if (remainingRetryTimes <= 0 && response == null) { + if (lastException != null) { + throw lastException; + } + throw new IllegalStateException("Received null from DescribeStream call."); + } } if (StreamStatus.ACTIVE.toString().equals(response.getStreamDescription().getStreamStatus()) @@ -195,54 +307,173 @@ public class KinesisProxy implements IKinesisProxyExtended { return null; } } - - /** - * {@inheritDoc} - */ - @Override - public Shard getShard(String shardId) { - if (this.listOfShardsSinceLastGet.get() == null) { - //Update this.listOfShardsSinceLastGet as needed. - this.getShardList(); + + private ListShardsResult listShards(final String nextToken) { + final ListShardsRequest request = new ListShardsRequest(); + request.setRequestCredentials(credentialsProvider.getCredentials()); + if (StringUtils.isEmpty(nextToken)) { + request.setStreamName(streamName); + } else { + request.setNextToken(nextToken); } - - for (Shard shard : listOfShardsSinceLastGet.get()) { - if (shard.getShardId().equals(shardId)) { - return shard; + ListShardsResult result = null; + LimitExceededException lastException = null; + int remainingRetries = this.maxListShardsRetryAttempts; + + while (result == null) { + try { + result = client.listShards(request); + } catch (LimitExceededException e) { + LOG.info("Got LimitExceededException when listing shards " + streamName + ". Backing off for " + + this.listShardsBackoffTimeInMillis + " millis."); + try { + Thread.sleep(this.listShardsBackoffTimeInMillis); + } catch (InterruptedException ie) { + LOG.debug("Stream " + streamName + " : Sleep was interrupted ", ie); + } + lastException = e; + } catch (ResourceInUseException e) { + LOG.info("Stream is not in Active/Updating status, returning null (wait until stream is in Active or" + + " Updating)"); + return null; + } + remainingRetries--; + if (remainingRetries <= 0 && result == null) { + if (lastException != null) { + throw lastException; + } + throw new IllegalStateException("Received null from ListShards call."); } } - LOG.warn("Cannot find the shard given the shardId " + shardId); - return null; + return result; } /** * {@inheritDoc} */ @Override - public List getShardList() { - List result = new ArrayList(); + public Shard getShard(String shardId) { + if (this.cachedShardMap == null) { + synchronized (this) { + if (this.cachedShardMap == null) { + this.getShardList(); + } + } + } - DescribeStreamResult response = null; - String lastShardId = null; + Shard shard = cachedShardMap.get(shardId); + if (shard == null) { + if (cacheMisses.incrementAndGet() > MAX_CACHE_MISSES_BEFORE_RELOAD || cacheNeedsTimeUpdate()) { + synchronized (this) { + shard = cachedShardMap.get(shardId); - do { - response = getStreamInfo(lastShardId); + // + // If after synchronizing we resolve the shard, it means someone else already got it for us. + // + if (shard == null) { + LOG.info("To many shard map cache misses or cache is out of date -- forcing a refresh"); + this.getShardList(); + shard = verifyAndLogShardAfterCacheUpdate(shardId); + cacheMisses.set(0); + } else { + // + // If someone else got us the shard go ahead and zero cache misses + // + cacheMisses.set(0); + } - if (response == null) { + } + } + } + + if (shard == null) { + String message = "Cannot find the shard given the shardId " + shardId + ". Cache misses: " + cacheMisses; + if (cacheMisses.get() % CACHE_MISS_WARNING_MODULUS == 0) { + LOG.warn(message); + } else { + LOG.debug(message); + } + } + return shard; + } + + private Shard verifyAndLogShardAfterCacheUpdate(String shardId) { + Shard shard = cachedShardMap.get(shardId); + if (shard == null) { + LOG.warn("Even after cache refresh shard '" + shardId + "' wasn't found. " + + "This could indicate a bigger problem"); + } + return shard; + } + + private boolean cacheNeedsTimeUpdate() { + if (lastCacheUpdateTime == null) { + return true; + } + Instant now = Instant.now(); + Duration cacheAge = Duration.between(lastCacheUpdateTime, now); + + String baseMessage = "Shard map cache is " + cacheAge + " > " + CACHE_MAX_ALLOWED_AGE + ". "; + if (cacheAge.compareTo(CACHE_MAX_ALLOWED_AGE) > 0) { + LOG.info(baseMessage + "Age exceeds limit -- Refreshing."); + return true; + } + LOG.debug(baseMessage + "Age doesn't exceed limit."); + return false; + } + + /** + * {@inheritDoc} + */ + @Override + public synchronized List getShardList() { + if (shardIterationState == null) { + shardIterationState = new ShardIterationState(); + } + + if (isKinesisClient) { + ListShardsResult result; + String nextToken = null; + + do { + result = listShards(nextToken); + + if (result == null) { + /* + * If listShards ever returns null, we should bail and return null. This indicates the stream is not + * in ACTIVE or UPDATING state and we may not have accurate/consistent information about the stream. + */ + return null; + } else { + shardIterationState.update(result.getShards()); + nextToken = result.getNextToken(); + } + } while (StringUtils.isNotEmpty(result.getNextToken())); + + } else { + DescribeStreamResult response; + + do { + response = getStreamInfo(shardIterationState.getLastShardId()); + + if (response == null) { /* * If getStreamInfo ever returns null, we should bail and return null. This indicates the stream is not * in ACTIVE or UPDATING state and we may not have accurate/consistent information about the stream. */ - return null; - } else { - List shards = response.getStreamDescription().getShards(); - result.addAll(shards); - lastShardId = shards.get(shards.size() - 1).getShardId(); - } - } while (response.getStreamDescription().isHasMoreShards()); - this.listOfShardsSinceLastGet.set(result); - return result; + return null; + } else { + shardIterationState.update(response.getStreamDescription().getShards()); + } + } while (response.getStreamDescription().isHasMoreShards()); + } + List shards = shardIterationState.getShards(); + this.cachedShardMap = shards.stream().collect(Collectors.toMap(Shard::getShardId, Function.identity())); + this.lastCacheUpdateTime = Instant.now(); + + shardIterationState = new ShardIterationState(); + return shards; } /** @@ -344,4 +575,26 @@ public class KinesisProxy implements IKinesisProxyExtended { return response; } + @Data + static class ShardIterationState { + + private List shards; + private String lastShardId; + + public ShardIterationState() { + shards = new ArrayList<>(); + } + + public void update(List shards) { + if (shards == null || shards.isEmpty()) { + return; + } + this.shards.addAll(shards); + Shard lastShard = shards.get(shards.size() - 1); + if (lastShardId == null || lastShardId.compareTo(lastShard.getShardId()) < 0) { + lastShardId = lastShard.getShardId(); + } + } + } + } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxyFactory.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxyFactory.java index 93df67e0..2a428b5e 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxyFactory.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxyFactory.java @@ -18,10 +18,15 @@ import com.amazonaws.ClientConfiguration; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.services.kinesis.AmazonKinesis; import com.amazonaws.services.kinesis.AmazonKinesisClient; +import com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisClientLibConfiguration; /** * Factory used for instantiating KinesisProxy objects (to fetch data from Kinesis). + * + * @deprecated Will be removed since proxy is created only once, we don't need a factory. There is no replacement for + * this class. Will be removed in the next major/minor release. */ +@Deprecated public class KinesisProxyFactory implements IKinesisProxyFactory { private final AWSCredentialsProvider credentialProvider; @@ -32,6 +37,8 @@ public class KinesisProxyFactory implements IKinesisProxyFactory { private final AmazonKinesis kinesisClient; private final long describeStreamBackoffTimeInMillis; private final int maxDescribeStreamRetryAttempts; + private final long listShardsBackoffTimeInMillis; + private final int maxListShardsRetryAttempts; /** * Constructor for creating a KinesisProxy factory, using the specified credentials provider and endpoint. @@ -41,12 +48,14 @@ public class KinesisProxyFactory implements IKinesisProxyFactory { */ public KinesisProxyFactory(AWSCredentialsProvider credentialProvider, String endpoint) { this(credentialProvider, new ClientConfiguration(), endpoint, defaultServiceName, defaultRegionId, - DEFAULT_DESCRIBE_STREAM_BACKOFF_MILLIS, DEFAULT_DESCRIBE_STREAM_RETRY_TIMES); + DEFAULT_DESCRIBE_STREAM_BACKOFF_MILLIS, DEFAULT_DESCRIBE_STREAM_RETRY_TIMES, + KinesisClientLibConfiguration.DEFAULT_LIST_SHARDS_BACKOFF_TIME_IN_MILLIS, + KinesisClientLibConfiguration.DEFAULT_MAX_LIST_SHARDS_RETRY_ATTEMPTS); } /** * Constructor for KinesisProxy factory using the client configuration to use when interacting with Kinesis. - * + * * @param credentialProvider credentials provider used to sign requests * @param clientConfig Client Configuration used when instantiating an AmazonKinesisClient * @param endpoint Amazon Kinesis endpoint to use @@ -55,7 +64,9 @@ public class KinesisProxyFactory implements IKinesisProxyFactory { ClientConfiguration clientConfig, String endpoint) { this(credentialProvider, clientConfig, endpoint, defaultServiceName, defaultRegionId, - DEFAULT_DESCRIBE_STREAM_BACKOFF_MILLIS, DEFAULT_DESCRIBE_STREAM_RETRY_TIMES); + DEFAULT_DESCRIBE_STREAM_BACKOFF_MILLIS, DEFAULT_DESCRIBE_STREAM_RETRY_TIMES, + KinesisClientLibConfiguration.DEFAULT_LIST_SHARDS_BACKOFF_TIME_IN_MILLIS, + KinesisClientLibConfiguration.DEFAULT_MAX_LIST_SHARDS_RETRY_ATTEMPTS); } /** @@ -65,7 +76,9 @@ public class KinesisProxyFactory implements IKinesisProxyFactory { * @param client AmazonKinesisClient used to fetch data from Kinesis */ public KinesisProxyFactory(AWSCredentialsProvider credentialProvider, AmazonKinesis client) { - this(credentialProvider, client, DEFAULT_DESCRIBE_STREAM_BACKOFF_MILLIS, DEFAULT_DESCRIBE_STREAM_RETRY_TIMES); + this(credentialProvider, client, DEFAULT_DESCRIBE_STREAM_BACKOFF_MILLIS, DEFAULT_DESCRIBE_STREAM_RETRY_TIMES, + KinesisClientLibConfiguration.DEFAULT_LIST_SHARDS_BACKOFF_TIME_IN_MILLIS, + KinesisClientLibConfiguration.DEFAULT_MAX_LIST_SHARDS_RETRY_ATTEMPTS); } /** @@ -85,13 +98,18 @@ public class KinesisProxyFactory implements IKinesisProxyFactory { String serviceName, String regionId, long describeStreamBackoffTimeInMillis, - int maxDescribeStreamRetryAttempts) { + int maxDescribeStreamRetryAttempts, + long listShardsBackoffTimeInMillis, + int maxListShardsRetryAttempts) { this(credentialProvider, buildClientSettingEndpoint(credentialProvider, clientConfig, endpoint, serviceName, regionId), - describeStreamBackoffTimeInMillis, maxDescribeStreamRetryAttempts); + describeStreamBackoffTimeInMillis, + maxDescribeStreamRetryAttempts, + listShardsBackoffTimeInMillis, + maxListShardsRetryAttempts); } @@ -106,14 +124,18 @@ public class KinesisProxyFactory implements IKinesisProxyFactory { KinesisProxyFactory(AWSCredentialsProvider credentialProvider, AmazonKinesis client, long describeStreamBackoffTimeInMillis, - int maxDescribeStreamRetryAttempts) { + int maxDescribeStreamRetryAttempts, + long listShardsBackoffTimeInMillis, + int maxListShardsRetryAttempts) { super(); this.kinesisClient = client; this.credentialProvider = credentialProvider; this.describeStreamBackoffTimeInMillis = describeStreamBackoffTimeInMillis; this.maxDescribeStreamRetryAttempts = maxDescribeStreamRetryAttempts; + this.listShardsBackoffTimeInMillis = listShardsBackoffTimeInMillis; + this.maxListShardsRetryAttempts = maxListShardsRetryAttempts; } - + /** * {@inheritDoc} */ @@ -123,15 +145,16 @@ public class KinesisProxyFactory implements IKinesisProxyFactory { credentialProvider, kinesisClient, describeStreamBackoffTimeInMillis, - maxDescribeStreamRetryAttempts); - + maxDescribeStreamRetryAttempts, + listShardsBackoffTimeInMillis, + maxListShardsRetryAttempts); } - + private static AmazonKinesisClient buildClientSettingEndpoint(AWSCredentialsProvider credentialProvider, - ClientConfiguration clientConfig, - String endpoint, - String serviceName, - String regionId) { + ClientConfiguration clientConfig, + String endpoint, + String serviceName, + String regionId) { AmazonKinesisClient client = new AmazonKinesisClient(credentialProvider, clientConfig); client.setEndpoint(endpoint); client.setSignerRegionOverride(regionId); diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/types/InitializationInput.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/types/InitializationInput.java index 8f044383..fce165f2 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/types/InitializationInput.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/types/InitializationInput.java @@ -1,16 +1,16 @@ /* - * Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.clientlibrary.types; @@ -23,6 +23,7 @@ public class InitializationInput { private String shardId; private ExtendedSequenceNumber extendedSequenceNumber; + private ExtendedSequenceNumber pendingCheckpointSequenceNumber; /** * Default constructor. @@ -71,4 +72,26 @@ public class InitializationInput { this.extendedSequenceNumber = extendedSequenceNumber; return this; } + + /** + * Get pending checkpoint {@link ExtendedSequenceNumber}. + * + * @return The {@link ExtendedSequenceNumber} in the shard for which a checkpoint is pending + */ + public ExtendedSequenceNumber getPendingCheckpointSequenceNumber() { + return pendingCheckpointSequenceNumber; + } + + /** + * Set pending checkpoint {@link ExtendedSequenceNumber}. + * + * @param pendingCheckpointSequenceNumber The {@link ExtendedSequenceNumber} in the shard for which a checkpoint + * is pending + * @return A reference to this updated object so that method calls can be chained together. + */ + public InitializationInput withPendingCheckpointSequenceNumber( + ExtendedSequenceNumber pendingCheckpointSequenceNumber) { + this.pendingCheckpointSequenceNumber = pendingCheckpointSequenceNumber; + return this; + } } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/types/Messages.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/types/Messages.java index 87545b7e..a467ee57 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/types/Messages.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/types/Messages.java @@ -1,3 +1,18 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + // Generated by the protocol buffer compiler. DO NOT EDIT! // source: messages.proto diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/types/ProcessRecordsInput.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/types/ProcessRecordsInput.java index f617e5e8..362af357 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/types/ProcessRecordsInput.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/types/ProcessRecordsInput.java @@ -1,30 +1,38 @@ /* - * Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.clientlibrary.types; +import java.time.Duration; +import java.time.Instant; import java.util.List; + import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer; import com.amazonaws.services.kinesis.model.Record; +import lombok.Getter; + /** * Container for the parameters to the IRecordProcessor's * {@link com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessor#processRecords( * ProcessRecordsInput processRecordsInput) processRecords} method. */ public class ProcessRecordsInput { - + @Getter + private Instant cacheEntryTime; + @Getter + private Instant cacheExitTime; private List records; private IRecordProcessorCheckpointer checkpointer; private Long millisBehindLatest; @@ -96,4 +104,21 @@ public class ProcessRecordsInput { this.millisBehindLatest = millisBehindLatest; return this; } + + public ProcessRecordsInput withCacheEntryTime(Instant cacheEntryTime) { + this.cacheEntryTime = cacheEntryTime; + return this; + } + + public ProcessRecordsInput withCacheExitTime(Instant cacheExitTime) { + this.cacheExitTime = cacheExitTime; + return this; + } + + public Duration getTimeSpentInCache() { + if (cacheEntryTime == null || cacheExitTime == null) { + return Duration.ZERO; + } + return Duration.between(cacheEntryTime, cacheExitTime); + } } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/types/ShutdownInput.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/types/ShutdownInput.java index c533a4da..368dd3d3 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/types/ShutdownInput.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/types/ShutdownInput.java @@ -1,16 +1,16 @@ /* - * Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.clientlibrary.types; diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/types/UserRecord.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/types/UserRecord.java index 2f60671a..068119d9 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/types/UserRecord.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/types/UserRecord.java @@ -23,7 +23,7 @@ import java.util.Arrays; import java.util.Date; import java.util.List; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/utils/NamedThreadFactory.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/utils/NamedThreadFactory.java index 29d8a7be..4be5a092 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/utils/NamedThreadFactory.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/utils/NamedThreadFactory.java @@ -1,3 +1,17 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ package com.amazonaws.services.kinesis.clientlibrary.utils; import java.util.concurrent.Executors; diff --git a/src/main/java/com/amazonaws/services/kinesis/leases/impl/KinesisClientLease.java b/src/main/java/com/amazonaws/services/kinesis/leases/impl/KinesisClientLease.java index b3a0ce6c..5f2d56b0 100644 --- a/src/main/java/com/amazonaws/services/kinesis/leases/impl/KinesisClientLease.java +++ b/src/main/java/com/amazonaws/services/kinesis/leases/impl/KinesisClientLease.java @@ -1,16 +1,16 @@ /* - * Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.leases.impl; @@ -27,6 +27,7 @@ import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber public class KinesisClientLease extends Lease { private ExtendedSequenceNumber checkpoint; + private ExtendedSequenceNumber pendingCheckpoint; private Long ownerSwitchesSinceCheckpoint = 0L; private Set parentShardIds = new HashSet(); @@ -37,16 +38,18 @@ public class KinesisClientLease extends Lease { public KinesisClientLease(KinesisClientLease other) { super(other); this.checkpoint = other.getCheckpoint(); + this.pendingCheckpoint = other.getPendingCheckpoint(); this.ownerSwitchesSinceCheckpoint = other.getOwnerSwitchesSinceCheckpoint(); this.parentShardIds.addAll(other.getParentShardIds()); } KinesisClientLease(String leaseKey, String leaseOwner, Long leaseCounter, UUID concurrencyToken, - Long lastCounterIncrementNanos, ExtendedSequenceNumber checkpoint, Long ownerSwitchesSinceCheckpoint, - Set parentShardIds) { + Long lastCounterIncrementNanos, ExtendedSequenceNumber checkpoint, ExtendedSequenceNumber pendingCheckpoint, + Long ownerSwitchesSinceCheckpoint, Set parentShardIds) { super(leaseKey, leaseOwner, leaseCounter, concurrencyToken, lastCounterIncrementNanos); this.checkpoint = checkpoint; + this.pendingCheckpoint = pendingCheckpoint; this.ownerSwitchesSinceCheckpoint = ownerSwitchesSinceCheckpoint; this.parentShardIds.addAll(parentShardIds); } @@ -64,6 +67,7 @@ public class KinesisClientLease extends Lease { setOwnerSwitchesSinceCheckpoint(casted.ownerSwitchesSinceCheckpoint); setCheckpoint(casted.checkpoint); + setPendingCheckpoint(casted.pendingCheckpoint); setParentShardIds(casted.parentShardIds); } @@ -75,6 +79,13 @@ public class KinesisClientLease extends Lease { return checkpoint; } + /** + * @return pending checkpoint, possibly null. + */ + public ExtendedSequenceNumber getPendingCheckpoint() { + return pendingCheckpoint; + } + /** * @return count of distinct lease holders between checkpoints. */ @@ -100,6 +111,15 @@ public class KinesisClientLease extends Lease { this.checkpoint = checkpoint; } + /** + * Sets pending checkpoint. + * + * @param pendingCheckpoint can be null + */ + public void setPendingCheckpoint(ExtendedSequenceNumber pendingCheckpoint) { + this.pendingCheckpoint = pendingCheckpoint; + } + /** * Sets ownerSwitchesSinceCheckpoint. * @@ -134,6 +154,7 @@ public class KinesisClientLease extends Lease { final int prime = 31; int result = super.hashCode(); result = prime * result + ((checkpoint == null) ? 0 : checkpoint.hashCode()); + result = pendingCheckpoint == null ? result : prime * result + pendingCheckpoint.hashCode(); result = prime * result + ((ownerSwitchesSinceCheckpoint == null) ? 0 : ownerSwitchesSinceCheckpoint.hashCode()); result = prime * result + ((parentShardIds == null) ? 0 : parentShardIds.hashCode()); @@ -154,6 +175,11 @@ public class KinesisClientLease extends Lease { return false; } else if (!checkpoint.equals(other.checkpoint)) return false; + if (pendingCheckpoint == null) { + if (other.pendingCheckpoint != null) + return false; + } else if (!pendingCheckpoint.equals(other.pendingCheckpoint)) + return false; if (ownerSwitchesSinceCheckpoint == null) { if (other.ownerSwitchesSinceCheckpoint != null) return false; diff --git a/src/main/java/com/amazonaws/services/kinesis/leases/impl/KinesisClientLeaseSerializer.java b/src/main/java/com/amazonaws/services/kinesis/leases/impl/KinesisClientLeaseSerializer.java index 0fad61ea..1234e164 100644 --- a/src/main/java/com/amazonaws/services/kinesis/leases/impl/KinesisClientLeaseSerializer.java +++ b/src/main/java/com/amazonaws/services/kinesis/leases/impl/KinesisClientLeaseSerializer.java @@ -1,16 +1,16 @@ /* - * Copyright 2012-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.leases.impl; @@ -26,6 +26,7 @@ import com.amazonaws.services.dynamodbv2.model.KeySchemaElement; import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber; import com.amazonaws.services.kinesis.leases.interfaces.ILeaseSerializer; import com.amazonaws.services.kinesis.leases.util.DynamoUtils; +import com.google.common.base.Strings; /** * An implementation of ILeaseSerializer for KinesisClientLease objects. @@ -35,6 +36,8 @@ public class KinesisClientLeaseSerializer implements ILeaseSerializer { private static final int DEFAULT_MAX_LEASES_FOR_WORKER = Integer.MAX_VALUE; private static final int DEFAULT_MAX_LEASES_TO_STEAL_AT_ONE_TIME = 1; - private static final ThreadFactory LEASE_COORDINATOR_THREAD_FACTORY = new NamedThreadFactory("LeaseCoordinator-"); - private static final ThreadFactory LEASE_RENEWAL_THREAD_FACTORY = new NamedThreadFactory("LeaseRenewer-"); - - // Package level access for testing. - static final int MAX_LEASE_RENEWAL_THREAD_COUNT = 20; - + private static final ThreadFactory LEASE_COORDINATOR_THREAD_FACTORY = new ThreadFactoryBuilder() + .setNameFormat("LeaseCoordinator-%04d").setDaemon(true).build(); + private static final ThreadFactory LEASE_RENEWAL_THREAD_FACTORY = new ThreadFactoryBuilder() + .setNameFormat("LeaseRenewer-%04d").setDaemon(true).build(); private final ILeaseRenewer leaseRenewer; private final ILeaseTaker leaseTaker; @@ -114,7 +115,8 @@ public class LeaseCoordinator { long epsilonMillis, IMetricsFactory metricsFactory) { this(leaseManager, workerIdentifier, leaseDurationMillis, epsilonMillis, - DEFAULT_MAX_LEASES_FOR_WORKER, DEFAULT_MAX_LEASES_TO_STEAL_AT_ONE_TIME, metricsFactory); + DEFAULT_MAX_LEASES_FOR_WORKER, DEFAULT_MAX_LEASES_TO_STEAL_AT_ONE_TIME, + KinesisClientLibConfiguration.DEFAULT_MAX_LEASE_RENEWAL_THREADS, metricsFactory); } /** @@ -134,8 +136,9 @@ public class LeaseCoordinator { long epsilonMillis, int maxLeasesForWorker, int maxLeasesToStealAtOneTime, + int maxLeaseRenewerThreadCount, IMetricsFactory metricsFactory) { - this.leaseRenewalThreadpool = getLeaseRenewalExecutorService(MAX_LEASE_RENEWAL_THREAD_COUNT); + this.leaseRenewalThreadpool = getLeaseRenewalExecutorService(maxLeaseRenewerThreadCount); this.leaseTaker = new LeaseTaker(leaseManager, workerIdentifier, leaseDurationMillis) .withMaxLeasesForWorker(maxLeasesForWorker) .withMaxLeasesToStealAtOneTime(maxLeasesToStealAtOneTime); @@ -366,6 +369,9 @@ public class LeaseCoordinator { * @return Executor service that should be used for lease renewal. */ private static ExecutorService getLeaseRenewalExecutorService(int maximumPoolSize) { - return Executors.newFixedThreadPool(maximumPoolSize, LEASE_RENEWAL_THREAD_FACTORY); + int coreLeaseCount = Math.max(maximumPoolSize / 4, 2); + + return new ThreadPoolExecutor(coreLeaseCount, maximumPoolSize, 60, TimeUnit.SECONDS, + new LinkedTransferQueue(), LEASE_RENEWAL_THREAD_FACTORY); } } diff --git a/src/main/java/com/amazonaws/services/kinesis/leases/impl/LeaseManager.java b/src/main/java/com/amazonaws/services/kinesis/leases/impl/LeaseManager.java index 226756eb..9dc2a4a3 100644 --- a/src/main/java/com/amazonaws/services/kinesis/leases/impl/LeaseManager.java +++ b/src/main/java/com/amazonaws/services/kinesis/leases/impl/LeaseManager.java @@ -1,16 +1,16 @@ /* - * Copyright 2012-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.leases.impl; @@ -19,6 +19,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import com.amazonaws.services.kinesis.leases.util.DynamoUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -386,7 +387,19 @@ public class LeaseManager implements ILeaseManager { + " because the lease counter was not " + lease.getLeaseCounter()); } - return false; + // If we had a spurious retry during the Dynamo update, then this conditional PUT failure + // might be incorrect. So, we get the item straight away and check if the lease owner + lease counter + // are what we expected. + String expectedOwner = lease.getLeaseOwner(); + Long expectedCounter = lease.getLeaseCounter() + 1; + T updatedLease = getLease(lease.getLeaseKey()); + if (updatedLease == null || !expectedOwner.equals(updatedLease.getLeaseOwner()) || + !expectedCounter.equals(updatedLease.getLeaseCounter())) { + return false; + } + + LOG.info("Detected spurious renewal failure for lease with key " + lease.getLeaseKey() + + ", but recovered"); } catch (AmazonClientException e) { throw convertAndRethrowExceptions("renew", lease.getLeaseKey(), e); } @@ -564,6 +577,7 @@ public class LeaseManager implements ILeaseManager { protected DependencyException convertAndRethrowExceptions(String operation, String leaseKey, AmazonClientException e) throws ProvisionedThroughputException, InvalidStateException { if (e instanceof ProvisionedThroughputExceededException) { + LOG.warn("Provisioned Throughput on the lease table has been exceeded. It's recommended that you increase the IOPs on the table. Failure to increase the IOPs may cause the application to not make progress."); throw new ProvisionedThroughputException(e); } else if (e instanceof ResourceNotFoundException) { // @formatter:on diff --git a/src/main/java/com/amazonaws/services/kinesis/metrics/impl/MetricsHelper.java b/src/main/java/com/amazonaws/services/kinesis/metrics/impl/MetricsHelper.java index 4599fbaa..bf104cff 100644 --- a/src/main/java/com/amazonaws/services/kinesis/metrics/impl/MetricsHelper.java +++ b/src/main/java/com/amazonaws/services/kinesis/metrics/impl/MetricsHelper.java @@ -72,13 +72,22 @@ public class MetricsHelper { * @param scope */ public static void setMetricsScope(IMetricsScope scope) { - if (currentScope.get() != null) { + if (isMetricsScopePresent()) { throw new RuntimeException(String.format( "Metrics scope is already set for the current thread %s", Thread.currentThread().getName())); } currentScope.set(scope); } + /** + * Checks if current metricsscope is present or not. + * + * @return true if metrics scope is present, else returns false + */ + public static boolean isMetricsScopePresent() { + return currentScope.get() != null; + } + /** * Unsets the metrics scope for the current thread. */ diff --git a/src/main/java/com/amazonaws/services/kinesis/multilang/MessageWriter.java b/src/main/java/com/amazonaws/services/kinesis/multilang/MessageWriter.java index b2ddbfe3..3310d248 100644 --- a/src/main/java/com/amazonaws/services/kinesis/multilang/MessageWriter.java +++ b/src/main/java/com/amazonaws/services/kinesis/multilang/MessageWriter.java @@ -26,13 +26,16 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import com.amazonaws.services.kinesis.clientlibrary.lib.worker.ShutdownReason; -import com.amazonaws.services.kinesis.clientlibrary.types.InitializationInput; -import com.amazonaws.services.kinesis.clientlibrary.types.ProcessRecordsInput; + import com.amazonaws.services.kinesis.multilang.messages.CheckpointMessage; import com.amazonaws.services.kinesis.multilang.messages.InitializeMessage; import com.amazonaws.services.kinesis.multilang.messages.Message; import com.amazonaws.services.kinesis.multilang.messages.ProcessRecordsMessage; import com.amazonaws.services.kinesis.multilang.messages.ShutdownMessage; +import com.amazonaws.services.kinesis.multilang.messages.ShutdownRequestedMessage; + +import com.amazonaws.services.kinesis.clientlibrary.types.InitializationInput; +import com.amazonaws.services.kinesis.clientlibrary.types.ProcessRecordsInput; import com.fasterxml.jackson.databind.ObjectMapper; /** @@ -145,6 +148,13 @@ class MessageWriter { return writeMessage(new ShutdownMessage(reason)); } + /** + * Writes a {@link ShutdownRequestedMessage} to the subprocess. + */ + Future writeShutdownRequestedMessage() { + return writeMessage(new ShutdownRequestedMessage()); + } + /** * Writes a {@link CheckpointMessage} to the subprocess. * diff --git a/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangDaemon.java b/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangDaemon.java index fdff4dc7..2c8d6909 100644 --- a/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangDaemon.java +++ b/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangDaemon.java @@ -1,25 +1,28 @@ /* - * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.multilang; import java.io.IOException; import java.io.PrintStream; + import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -140,11 +143,26 @@ public class MultiLangDaemon implements Callable { ExecutorService executorService = config.getExecutorService(); // Daemon - MultiLangDaemon daemon = new MultiLangDaemon( + final MultiLangDaemon daemon = new MultiLangDaemon( config.getKinesisClientLibConfiguration(), config.getRecordProcessorFactory(), executorService); + final long shutdownGraceMillis = config.getKinesisClientLibConfiguration().getShutdownGraceMillis(); + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + LOG.info("Process terminanted, will initiate shutdown."); + try { + Future fut = daemon.worker.requestShutdown(); + fut.get(shutdownGraceMillis, TimeUnit.MILLISECONDS); + LOG.info("Process shutdown is complete."); + } catch (InterruptedException | ExecutionException | TimeoutException e) { + LOG.error("Encountered an error during shutdown.", e); + } + } + }); + Future future = executorService.submit(daemon); try { System.exit(future.get()); diff --git a/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangDaemonConfig.java b/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangDaemonConfig.java index f191eedc..58f475a3 100644 --- a/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangDaemonConfig.java +++ b/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangDaemonConfig.java @@ -1,24 +1,34 @@ /* - * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.multilang; +import com.amazonaws.ClientConfiguration; +import com.amazonaws.services.kinesis.clientlibrary.config.KinesisClientLibConfigurator; +import com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisClientLibConfiguration; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + import java.io.File; -import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.io.FileInputStream; +import java.net.URI; +import java.net.URISyntaxException; import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; @@ -26,13 +36,6 @@ import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import com.amazonaws.services.kinesis.clientlibrary.config.KinesisClientLibConfigurator; -import com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisClientLibConfiguration; -import com.google.common.util.concurrent.ThreadFactoryBuilder; - /** * This class captures the configuration needed to run the MultiLangDaemon. */ @@ -47,6 +50,10 @@ public class MultiLangDaemonConfig { private static final String PROP_PROCESSING_LANGUAGE = "processingLanguage"; private static final String PROP_MAX_ACTIVE_THREADS = "maxActiveThreads"; + public static final String PROXY_HOST_PROP = "http.proxyHost"; + public static final String PROXY_PORT_PROP = "http.proxyPort"; + public static final String HTTP_PROXY_ENV_VAR = "HTTP_PROXY"; + private KinesisClientLibConfiguration kinesisClientLibConfig; private ExecutorService executorService; @@ -55,9 +62,9 @@ public class MultiLangDaemonConfig { /** * Constructor. - * + * * @param propertiesFile The location of the properties file. - * @throws IOException Thrown when the properties file can't be accessed. + * @throws IOException Thrown when the properties file can't be accessed. * @throws IllegalArgumentException Thrown when the contents of the properties file are not as expected. */ public MultiLangDaemonConfig(String propertiesFile) throws IOException, IllegalArgumentException { @@ -65,55 +72,93 @@ public class MultiLangDaemonConfig { } /** - * * @param propertiesFile The location of the properties file. - * @param classLoader A classloader, useful if trying to programmatically configure with the daemon, such as in a - * unit test. - * @throws IOException Thrown when the properties file can't be accessed. + * @param classLoader A classloader, useful if trying to programmatically configure with the daemon, such as in a + * unit test. + * @throws IOException Thrown when the properties file can't be accessed. * @throws IllegalArgumentException Thrown when the contents of the properties file are not as expected. */ - public MultiLangDaemonConfig(String propertiesFile, ClassLoader classLoader) throws IOException, - IllegalArgumentException { + public MultiLangDaemonConfig(String propertiesFile, ClassLoader classLoader) + throws IOException, IllegalArgumentException { this(propertiesFile, classLoader, new KinesisClientLibConfigurator()); } /** - * * @param propertiesFile The location of the properties file. - * @param classLoader A classloader, useful if trying to programmatically configure with the daemon, such as in a - * unit test. - * @param configurator A configurator to use. - * @throws IOException Thrown when the properties file can't be accessed. + * @param classLoader A classloader, useful if trying to programmatically configure with the daemon, such as in a + * unit test. + * @param configurator A configurator to use. + * @throws IOException Thrown when the properties file can't be accessed. * @throws IllegalArgumentException Thrown when the contents of the properties file are not as expected. */ - public MultiLangDaemonConfig(String propertiesFile, - ClassLoader classLoader, + public MultiLangDaemonConfig(String propertiesFile, ClassLoader classLoader, KinesisClientLibConfigurator configurator) throws IOException, IllegalArgumentException { Properties properties = loadProperties(classLoader, propertiesFile); if (!validateProperties(properties)) { - throw new IllegalArgumentException("Must provide an executable name in the properties file, " - + "e.g. executableName = sampleapp.py"); + throw new IllegalArgumentException( + "Must provide an executable name in the properties file, " + "e.g. executableName = sampleapp.py"); } String executableName = properties.getProperty(PROP_EXECUTABLE_NAME); String processingLanguage = properties.getProperty(PROP_PROCESSING_LANGUAGE); + ClientConfiguration clientConfig = buildClientConfig(properties); + + kinesisClientLibConfig = configurator.getConfiguration(properties).withKinesisClientConfig(clientConfig) + .withCloudWatchClientConfig(clientConfig).withDynamoDBClientConfig(clientConfig); - kinesisClientLibConfig = configurator.getConfiguration(properties); executorService = buildExecutorService(properties); - recordProcessorFactory = new MultiLangRecordProcessorFactory(executableName, executorService); + recordProcessorFactory = new MultiLangRecordProcessorFactory(executableName, executorService, + kinesisClientLibConfig); LOG.info("Running " + kinesisClientLibConfig.getApplicationName() + " to process stream " + kinesisClientLibConfig.getStreamName() + " with executable " + executableName); prepare(processingLanguage); } + private ClientConfiguration buildClientConfig(Properties properties) { + ClientConfiguration clientConfig = new ClientConfiguration(); + String proxyHost = null; + int proxyPort = 0; + + if (properties.getProperty(PROXY_HOST_PROP) != null) { + LOG.debug("Getting proxy info from properties file."); + + proxyHost = properties.getProperty(PROXY_HOST_PROP); + proxyPort = Integer.parseInt(properties.getProperty(PROXY_PORT_PROP)); + } else if (System.getProperty(PROXY_HOST_PROP) != null) { + LOG.debug("Getting proxy info from java system properties"); + + proxyHost = System.getProperty(PROXY_HOST_PROP); + proxyPort = Integer.parseInt(System.getProperty(PROXY_PORT_PROP)); + } else if (System.getenv(HTTP_PROXY_ENV_VAR) != null) { + LOG.debug("Getting proxy info environment settings"); + + try { + URI proxyAddr = new URI(System.getenv(HTTP_PROXY_ENV_VAR)); + + proxyHost = proxyAddr.getHost(); + proxyPort = proxyAddr.getPort(); + } catch (URISyntaxException e) { + LOG.error("System proxy not set correctly", e); + } + } + + if (StringUtils.isNotEmpty(proxyHost) && proxyPort > 0) { + clientConfig = clientConfig.withProxyHost(proxyHost).withProxyPort(proxyPort); + } else { + LOG.debug("Not configuring proxy as none specified"); + } + + return clientConfig; + } + private void prepare(String processingLanguage) { // Ensure the JVM will refresh the cached IP values of AWS resources (e.g. service endpoints). java.security.Security.setProperty("networkaddress.cache.ttl", "60"); LOG.info("Using workerId: " + kinesisClientLibConfig.getWorkerIdentifier()); - LOG.info("Using credentials with access key id: " - + kinesisClientLibConfig.getKinesisCredentialsProvider().getCredentials().getAWSAccessKeyId()); + LOG.info("Using credentials with access key id: " + kinesisClientLibConfig.getKinesisCredentialsProvider() + .getCredentials().getAWSAccessKeyId()); StringBuilder userAgent = new StringBuilder(KinesisClientLibConfiguration.KINESIS_CLIENT_LIB_USER_AGENT); userAgent.append(" "); @@ -187,7 +232,6 @@ public class MultiLangDaemonConfig { } /** - * * @return A KinesisClientLibConfiguration object based on the properties file provided. */ public KinesisClientLibConfiguration getKinesisClientLibConfiguration() { @@ -195,7 +239,6 @@ public class MultiLangDaemonConfig { } /** - * * @return An executor service based on the properties file provided. */ public ExecutorService getExecutorService() { @@ -203,7 +246,6 @@ public class MultiLangDaemonConfig { } /** - * * @return A MultiLangRecordProcessorFactory based on the properties file provided. */ public MultiLangRecordProcessorFactory getRecordProcessorFactory() { diff --git a/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangProtocol.java b/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangProtocol.java index 64c7829f..7a809289 100644 --- a/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangProtocol.java +++ b/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangProtocol.java @@ -1,24 +1,22 @@ /* - * Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.multilang; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; - import com.amazonaws.services.kinesis.clientlibrary.exceptions.InvalidStateException; import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer; +import com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisClientLibConfiguration; import com.amazonaws.services.kinesis.clientlibrary.lib.worker.ShutdownReason; import com.amazonaws.services.kinesis.clientlibrary.types.InitializationInput; import com.amazonaws.services.kinesis.clientlibrary.types.ProcessRecordsInput; @@ -27,10 +25,16 @@ import com.amazonaws.services.kinesis.multilang.messages.InitializeMessage; import com.amazonaws.services.kinesis.multilang.messages.Message; import com.amazonaws.services.kinesis.multilang.messages.ProcessRecordsMessage; import com.amazonaws.services.kinesis.multilang.messages.ShutdownMessage; +import com.amazonaws.services.kinesis.multilang.messages.ShutdownRequestedMessage; import com.amazonaws.services.kinesis.multilang.messages.StatusMessage; - import lombok.extern.apachecommons.CommonsLog; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + /** * An implementation of the multi language protocol. */ @@ -40,10 +44,11 @@ class MultiLangProtocol { private MessageReader messageReader; private MessageWriter messageWriter; private final InitializationInput initializationInput; + private KinesisClientLibConfiguration configuration; /** * Constructor. - * + * * @param messageReader * A message reader. * @param messageWriter @@ -52,16 +57,17 @@ class MultiLangProtocol { * information about the shard this processor is starting to process */ MultiLangProtocol(MessageReader messageReader, MessageWriter messageWriter, - InitializationInput initializationInput) { + InitializationInput initializationInput, KinesisClientLibConfiguration configuration) { this.messageReader = messageReader; this.messageWriter = messageWriter; this.initializationInput = initializationInput; + this.configuration = configuration; } /** * Writes an {@link InitializeMessage} to the child process's STDIN and waits for the child process to respond with * a {@link StatusMessage} on its STDOUT. - * + * * @return Whether or not this operation succeeded. */ boolean initialize() { @@ -76,7 +82,7 @@ class MultiLangProtocol { /** * Writes a {@link ProcessRecordsMessage} to the child process's STDIN and waits for the child process to respond * with a {@link StatusMessage} on its STDOUT. - * + * * @param processRecordsInput * The records, and associated metadata, to process. * @return Whether or not this operation succeeded. @@ -89,7 +95,7 @@ class MultiLangProtocol { /** * Writes a {@link ShutdownMessage} to the child process's STDIN and waits for the child process to respond with a * {@link StatusMessage} on its STDOUT. - * + * * @param checkpointer A checkpointer. * @param reason Why this processor is being shutdown. * @return Whether or not this operation succeeded. @@ -99,6 +105,18 @@ class MultiLangProtocol { return waitForStatusMessage(ShutdownMessage.ACTION, checkpointer, writeFuture); } + /** + * Writes a {@link ShutdownRequestedMessage} to the child process's STDIN and waits for the child process to respond with a + * {@link StatusMessage} on its STDOUT. + * + * @param checkpointer A checkpointer. + * @return Whether or not this operation succeeded. + */ + boolean shutdownRequested(IRecordProcessorCheckpointer checkpointer) { + Future writeFuture = messageWriter.writeShutdownRequestedMessage(); + return waitForStatusMessage(ShutdownRequestedMessage.ACTION, checkpointer, writeFuture); + } + /** * Waits for a {@link StatusMessage} for a particular action. If a {@link CheckpointMessage} is received, then this * method will attempt to checkpoint with the provided {@link IRecordProcessorCheckpointer}. This method returns @@ -106,7 +124,7 @@ class MultiLangProtocol { * all communications with the child process regarding checkpointing were successful. Note that whether or not the * checkpointing itself was successful is not the concern of this method. This method simply cares whether it was * able to successfully communicate the results of its attempts to checkpoint. - * + * * @param action * What action is being waited on. * @param checkpointer @@ -137,44 +155,75 @@ class MultiLangProtocol { /** * Waits for status message and verifies it against the expectation - * + * * @param action * What action is being waited on. * @param checkpointer * the original process records request * @return Whether or not this operation succeeded. */ - private boolean waitForStatusMessage(String action, IRecordProcessorCheckpointer checkpointer) { - StatusMessage statusMessage = null; - while (statusMessage == null) { + boolean waitForStatusMessage(String action, IRecordProcessorCheckpointer checkpointer) { + Optional statusMessage = Optional.empty(); + while (!statusMessage.isPresent()) { Future future = this.messageReader.getNextMessageFromSTDOUT(); - try { - Message message = future.get(); - // Note that instanceof doubles as a check against a value being null - if (message instanceof CheckpointMessage) { - boolean checkpointWriteSucceeded = checkpoint((CheckpointMessage) message, checkpointer).get(); - if (!checkpointWriteSucceeded) { - return false; - } - } else if (message instanceof StatusMessage) { - statusMessage = (StatusMessage) message; - } - } catch (InterruptedException e) { - log.error(String.format("Interrupted while waiting for %s message for shard %s", action, - initializationInput.getShardId())); - return false; - } catch (ExecutionException e) { - log.error(String.format("Failed to get status message for %s action for shard %s", action, - initializationInput.getShardId()), e); + Optional message = configuration.getTimeoutInSeconds() + .map(second -> futureMethod(() -> future.get(second, TimeUnit.SECONDS), action)) + .orElse(futureMethod(future::get, action)); + + if (!message.isPresent()) { return false; } + + Optional checkpointFailed = message.filter(m -> m instanceof CheckpointMessage ) + .map(m -> (CheckpointMessage) m) + .flatMap(m -> futureMethod(() -> checkpoint(m, checkpointer).get(), "Checkpoint")) + .map(checkpointSuccess -> !checkpointSuccess); + + if (checkpointFailed.orElse(false)) { + return false; + } + + statusMessage = message.filter(m -> m instanceof StatusMessage).map(m -> (StatusMessage) m ); } - return this.validateStatusMessage(statusMessage, action); + return this.validateStatusMessage(statusMessage.get(), action); + } + + private interface FutureMethod { + T get() throws InterruptedException, TimeoutException, ExecutionException; + } + + private Optional futureMethod(FutureMethod fm, String action) { + try { + return Optional.of(fm.get()); + } catch (InterruptedException e) { + log.error(String.format("Interrupted while waiting for %s message for shard %s", action, + initializationInput.getShardId()), e); + } catch (ExecutionException e) { + log.error(String.format("Failed to get status message for %s action for shard %s", action, + initializationInput.getShardId()), e); + } catch (TimeoutException e) { + log.error(String.format("Timedout to get status message for %s action for shard %s. Terminating...", + action, + initializationInput.getShardId()), + e); + haltJvm(1); + } + return Optional.empty(); + } + + /** + * This method is used to halt the JVM. Use this method with utmost caution, since this method will kill the JVM + * without calling the Shutdown hooks. + * + * @param exitStatus The exit status with which the JVM is to be halted. + */ + protected void haltJvm(int exitStatus) { + Runtime.getRuntime().halt(exitStatus); } /** * Utility for confirming that the status message is for the provided action. - * + * * @param statusMessage The status of the child process. * @param action The action that was being waited on. * @return Whether or not this operation succeeded. @@ -192,7 +241,7 @@ class MultiLangProtocol { * provided {@link CheckpointMessage}. If no sequence number is provided, i.e. the sequence number is null, then * this method will call {@link IRecordProcessorCheckpointer#checkpoint()}. The method returns a future representing * the attempt to write the result of this checkpoint attempt to the child process. - * + * * @param checkpointMessage A checkpoint message. * @param checkpointer A checkpointer. * @return Whether or not this operation succeeded. diff --git a/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangRecordProcessor.java b/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangRecordProcessor.java index 9d76af54..1261c06a 100644 --- a/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangRecordProcessor.java +++ b/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangRecordProcessor.java @@ -1,16 +1,16 @@ /* - * Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.multilang; @@ -20,6 +20,11 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import com.amazonaws.services.kinesis.clientlibrary.exceptions.InvalidStateException; +import com.amazonaws.services.kinesis.clientlibrary.exceptions.ShutdownException; +import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer; +import com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IShutdownNotificationAware; +import com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisClientLibConfiguration; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -29,13 +34,14 @@ import com.amazonaws.services.kinesis.clientlibrary.types.ProcessRecordsInput; import com.amazonaws.services.kinesis.clientlibrary.types.ShutdownInput; import com.fasterxml.jackson.databind.ObjectMapper; + /** * A record processor that manages creating a child process that implements the multi language protocol and connecting * that child process's input and outputs to a {@link MultiLangProtocol} object and calling the appropriate methods on * that object when its corresponding {@link #initialize}, {@link #processRecords}, and {@link #shutdown} methods are * called. */ -public class MultiLangRecordProcessor implements IRecordProcessor { +public class MultiLangRecordProcessor implements IRecordProcessor, IShutdownNotificationAware { private static final Log LOG = LogFactory.getLog(MultiLangRecordProcessor.class); private static final int EXIT_VALUE = 1; @@ -60,6 +66,8 @@ public class MultiLangRecordProcessor implements IRecordProcessor { private MultiLangProtocol protocol; + private KinesisClientLibConfiguration configuration; + @Override public void initialize(InitializationInput initializationInput) { try { @@ -82,7 +90,7 @@ public class MultiLangRecordProcessor implements IRecordProcessor { // Submit the error reader for execution stderrReadTask = executorService.submit(readSTDERRTask); - protocol = new MultiLangProtocol(messageReader, messageWriter, initializationInput); + protocol = new MultiLangProtocol(messageReader, messageWriter, initializationInput, configuration); if (!protocol.initialize()) { throw new RuntimeException("Failed to initialize child process"); } @@ -136,6 +144,20 @@ public class MultiLangRecordProcessor implements IRecordProcessor { } } + @Override + public void shutdownRequested(IRecordProcessorCheckpointer checkpointer) { + LOG.info("Shutdown is requested."); + if (!initialized) { + LOG.info("Record processor was not initialized so no need to initiate a final checkpoint."); + return; + } + LOG.info("Requesting a checkpoint on shutdown notification."); + if (!protocol.shutdownRequested(checkpointer)) { + LOG.error("Child process failed to complete shutdown notification."); + } + } + + /** * Used to tell whether the processor has been shutdown already. */ @@ -154,9 +176,9 @@ public class MultiLangRecordProcessor implements IRecordProcessor { * An obejct mapper. */ MultiLangRecordProcessor(ProcessBuilder processBuilder, ExecutorService executorService, - ObjectMapper objectMapper) { + ObjectMapper objectMapper, KinesisClientLibConfiguration configuration) { this(processBuilder, executorService, objectMapper, new MessageWriter(), new MessageReader(), - new DrainChildSTDERRTask()); + new DrainChildSTDERRTask(), configuration); } /** @@ -176,13 +198,16 @@ public class MultiLangRecordProcessor implements IRecordProcessor { * Error reader to read from child process's stderr */ MultiLangRecordProcessor(ProcessBuilder processBuilder, ExecutorService executorService, ObjectMapper objectMapper, - MessageWriter messageWriter, MessageReader messageReader, DrainChildSTDERRTask readSTDERRTask) { + MessageWriter messageWriter, MessageReader messageReader, DrainChildSTDERRTask readSTDERRTask, + KinesisClientLibConfiguration configuration) { this.executorService = executorService; this.processBuilder = processBuilder; this.objectMapper = objectMapper; this.messageWriter = messageWriter; this.messageReader = messageReader; this.readSTDERRTask = readSTDERRTask; + this.configuration = configuration; + this.state = ProcessState.ACTIVE; } diff --git a/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangRecordProcessorFactory.java b/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangRecordProcessorFactory.java index e55217a6..eadb1f6d 100644 --- a/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangRecordProcessorFactory.java +++ b/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangRecordProcessorFactory.java @@ -1,21 +1,22 @@ /* - * Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.multilang; import java.util.concurrent.ExecutorService; +import com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisClientLibConfiguration; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -39,12 +40,15 @@ public class MultiLangRecordProcessorFactory implements IRecordProcessorFactory private final ExecutorService executorService; + private final KinesisClientLibConfiguration configuration; + /** * @param command The command that will do processing for this factory's record processors. * @param executorService An executor service to use while processing inputs and outputs of the child process. */ - public MultiLangRecordProcessorFactory(String command, ExecutorService executorService) { - this(command, executorService, new ObjectMapper()); + public MultiLangRecordProcessorFactory(String command, ExecutorService executorService, + KinesisClientLibConfiguration configuration) { + this(command, executorService, new ObjectMapper(), configuration); } /** @@ -52,11 +56,13 @@ public class MultiLangRecordProcessorFactory implements IRecordProcessorFactory * @param executorService An executor service to use while processing inputs and outputs of the child process. * @param objectMapper An object mapper used to convert messages to json to be written to the child process */ - public MultiLangRecordProcessorFactory(String command, ExecutorService executorService, ObjectMapper objectMapper) { + public MultiLangRecordProcessorFactory(String command, ExecutorService executorService, ObjectMapper objectMapper, + KinesisClientLibConfiguration configuration) { this.command = command; this.commandArray = command.split(COMMAND_DELIMETER_REGEX); this.executorService = executorService; this.objectMapper = objectMapper; + this.configuration = configuration; } @Override @@ -65,7 +71,8 @@ public class MultiLangRecordProcessorFactory implements IRecordProcessorFactory /* * Giving ProcessBuilder the command as an array of Strings allows users to specify command line arguments. */ - return new MultiLangRecordProcessor(new ProcessBuilder(commandArray), executorService, this.objectMapper); + return new MultiLangRecordProcessor(new ProcessBuilder(commandArray), executorService, this.objectMapper, + this.configuration); } String[] getCommandArray() { diff --git a/src/main/java/com/amazonaws/services/kinesis/multilang/messages/CheckpointMessage.java b/src/main/java/com/amazonaws/services/kinesis/multilang/messages/CheckpointMessage.java index 5cdc02bd..f38980ba 100644 --- a/src/main/java/com/amazonaws/services/kinesis/multilang/messages/CheckpointMessage.java +++ b/src/main/java/com/amazonaws/services/kinesis/multilang/messages/CheckpointMessage.java @@ -1,16 +1,16 @@ /* - * Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.multilang.messages; diff --git a/src/main/java/com/amazonaws/services/kinesis/multilang/messages/InitializeMessage.java b/src/main/java/com/amazonaws/services/kinesis/multilang/messages/InitializeMessage.java index 3795e57e..cc6be56f 100644 --- a/src/main/java/com/amazonaws/services/kinesis/multilang/messages/InitializeMessage.java +++ b/src/main/java/com/amazonaws/services/kinesis/multilang/messages/InitializeMessage.java @@ -1,16 +1,16 @@ /* - * Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.multilang.messages; diff --git a/src/main/java/com/amazonaws/services/kinesis/multilang/messages/JsonFriendlyRecord.java b/src/main/java/com/amazonaws/services/kinesis/multilang/messages/JsonFriendlyRecord.java index 600489fe..19100993 100644 --- a/src/main/java/com/amazonaws/services/kinesis/multilang/messages/JsonFriendlyRecord.java +++ b/src/main/java/com/amazonaws/services/kinesis/multilang/messages/JsonFriendlyRecord.java @@ -1,16 +1,16 @@ /* - * Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.multilang.messages; diff --git a/src/main/java/com/amazonaws/services/kinesis/multilang/messages/Message.java b/src/main/java/com/amazonaws/services/kinesis/multilang/messages/Message.java index 766cdac0..7470b8e2 100644 --- a/src/main/java/com/amazonaws/services/kinesis/multilang/messages/Message.java +++ b/src/main/java/com/amazonaws/services/kinesis/multilang/messages/Message.java @@ -1,16 +1,16 @@ /* - * Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.multilang.messages; @@ -23,11 +23,14 @@ import com.fasterxml.jackson.databind.ObjectMapper; * Abstract class for all messages that are sent to the client's process. */ @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "action") -@JsonSubTypes({ @Type(value = CheckpointMessage.class, name = CheckpointMessage.ACTION), +@JsonSubTypes({ + @Type(value = CheckpointMessage.class, name = CheckpointMessage.ACTION), @Type(value = InitializeMessage.class, name = InitializeMessage.ACTION), @Type(value = ProcessRecordsMessage.class, name = ProcessRecordsMessage.ACTION), @Type(value = ShutdownMessage.class, name = ShutdownMessage.ACTION), - @Type(value = StatusMessage.class, name = StatusMessage.ACTION), }) + @Type(value = StatusMessage.class, name = StatusMessage.ACTION), + @Type(value = ShutdownRequestedMessage.class, name = ShutdownRequestedMessage.ACTION), +}) public abstract class Message { private ObjectMapper mapper = new ObjectMapper();; diff --git a/src/main/java/com/amazonaws/services/kinesis/multilang/messages/ProcessRecordsMessage.java b/src/main/java/com/amazonaws/services/kinesis/multilang/messages/ProcessRecordsMessage.java index 9e382b93..12371eb8 100644 --- a/src/main/java/com/amazonaws/services/kinesis/multilang/messages/ProcessRecordsMessage.java +++ b/src/main/java/com/amazonaws/services/kinesis/multilang/messages/ProcessRecordsMessage.java @@ -1,16 +1,16 @@ /* - * Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.multilang.messages; diff --git a/src/main/java/com/amazonaws/services/kinesis/multilang/messages/ShutdownRequestedMessage.java b/src/main/java/com/amazonaws/services/kinesis/multilang/messages/ShutdownRequestedMessage.java new file mode 100644 index 00000000..409cbce4 --- /dev/null +++ b/src/main/java/com/amazonaws/services/kinesis/multilang/messages/ShutdownRequestedMessage.java @@ -0,0 +1,31 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package com.amazonaws.services.kinesis.multilang.messages; + +/** + * A message to indicate to the client's process that shutdown is requested. + */ +public class ShutdownRequestedMessage extends Message { + /** + * The name used for the action field in {@link Message}. + */ + public static final String ACTION = "shutdownRequested"; + + /** + * Convenience constructor. + */ + public ShutdownRequestedMessage() { + } +} diff --git a/src/test/java/com/amazonaws/services/dynamodbv2/streamsadapter/AmazonDynamoDBStreamsAdapterClient.java b/src/test/java/com/amazonaws/services/dynamodbv2/streamsadapter/AmazonDynamoDBStreamsAdapterClient.java new file mode 100644 index 00000000..b654ca00 --- /dev/null +++ b/src/test/java/com/amazonaws/services/dynamodbv2/streamsadapter/AmazonDynamoDBStreamsAdapterClient.java @@ -0,0 +1,26 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazonaws.services.dynamodbv2.streamsadapter; + +import com.amazonaws.services.kinesis.AmazonKinesis; +import com.amazonaws.services.kinesis.AmazonKinesisClient; + +/** + * This class is only used for testing purposes, to make sure that the correct calls are made while using DynamoDB + * streams. + */ +public class AmazonDynamoDBStreamsAdapterClient extends AmazonKinesisClient { +} diff --git a/src/test/java/com/amazonaws/services/dynamodbv2/streamsadapter/AmazonDynamoDBStreamsAdapterClientChild.java b/src/test/java/com/amazonaws/services/dynamodbv2/streamsadapter/AmazonDynamoDBStreamsAdapterClientChild.java new file mode 100644 index 00000000..95dc1607 --- /dev/null +++ b/src/test/java/com/amazonaws/services/dynamodbv2/streamsadapter/AmazonDynamoDBStreamsAdapterClientChild.java @@ -0,0 +1,23 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazonaws.services.dynamodbv2.streamsadapter; + +/** + * This class is only used for testing purposes, to make sure that the correct calls are made while using DynamoDB + * streams. + */ +public class AmazonDynamoDBStreamsAdapterClientChild extends AmazonDynamoDBStreamsAdapterClient { +} diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/config/DatePropertyValueDecoderTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/config/DatePropertyValueDecoderTest.java new file mode 100644 index 00000000..df405978 --- /dev/null +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/config/DatePropertyValueDecoderTest.java @@ -0,0 +1,53 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package com.amazonaws.services.kinesis.clientlibrary.config; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.util.Date; + +import org.junit.Test; + +import com.amazonaws.services.kinesis.clientlibrary.config.DatePropertyValueDecoder; + +public class DatePropertyValueDecoderTest { + + private DatePropertyValueDecoder decoder = new DatePropertyValueDecoder(); + + private static final String TEST_VALUE = "1527267472"; + + @Test + public void testNumericValue() { + Date timestamp = decoder.decodeValue(TEST_VALUE); + assertEquals(timestamp.getClass(), Date.class); + assertEquals(timestamp, new Date(Long.parseLong(TEST_VALUE) * 1000L)); + } + + @Test(expected = IllegalArgumentException.class) + public void testEmptyValue() { + Date timestamp = decoder.decodeValue(""); + } + + @Test(expected = IllegalArgumentException.class) + public void testNullValue() { + Date timestamp = decoder.decodeValue(null); + } + + @Test(expected = IllegalArgumentException.class) + public void testNonNumericValue() { + Date timestamp = decoder.decodeValue("123abc"); + } +} diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/config/KinesisClientLibConfiguratorTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/config/KinesisClientLibConfiguratorTest.java index cbdd0a2d..19ba20a1 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/config/KinesisClientLibConfiguratorTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/config/KinesisClientLibConfiguratorTest.java @@ -1,16 +1,16 @@ /* - * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.clientlibrary.config; @@ -22,9 +22,11 @@ import static org.junit.Assert.fail; import java.io.ByteArrayInputStream; import java.io.InputStream; +import java.util.Date; +import java.util.Optional; import java.util.Set; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.junit.Test; import com.amazonaws.auth.AWSCredentials; @@ -60,6 +62,8 @@ public class KinesisClientLibConfiguratorTest { assertEquals(config.getApplicationName(), "b"); assertEquals(config.getStreamName(), "a"); assertEquals(config.getWorkerIdentifier(), "123"); + assertEquals(config.getMaxGetRecordsThreadPool(), Optional.empty()); + assertEquals(config.getRetryGetRecordsInSeconds(), Optional.empty()); } @Test @@ -107,7 +111,9 @@ public class KinesisClientLibConfiguratorTest { "workerId = w123", "maxRecords = 10", "metricsMaxQueueSize = 20", - "applicationName = kinesis" + "applicationName = kinesis", + "retryGetRecordsInSeconds = 2", + "maxGetRecordsThreadPool = 1" }, '\n')); assertEquals(config.getApplicationName(), "kinesis"); @@ -115,6 +121,8 @@ public class KinesisClientLibConfiguratorTest { assertEquals(config.getWorkerIdentifier(), "w123"); assertEquals(config.getMaxRecords(), 10); assertEquals(config.getMetricsMaxQueueSize(), 20); + assertEquals(config.getRetryGetRecordsInSeconds(), Optional.of(2)); + assertEquals(config.getMaxGetRecordsThreadPool(), Optional.of(1)); } @Test @@ -136,6 +144,20 @@ public class KinesisClientLibConfiguratorTest { assertTrue(config.shouldValidateSequenceNumberBeforeCheckpointing()); } + @Test + public void testWithDateVariables() { + KinesisClientLibConfiguration config = + getConfiguration(StringUtils.join(new String[] { + "streamName = a", + "applicationName = b", + "AWSCredentialsProvider = ABCD, " + credentialName1, + "timestampAtInitialPositionInStream = 1527267472" + }, '\n')); + + assertEquals(config.getTimestampAtInitialPositionInStream(), + new Date(1527267472 * 1000L)); + } + @Test public void testWithStringVariables() { KinesisClientLibConfiguration config = @@ -182,6 +204,49 @@ public class KinesisClientLibConfiguratorTest { }, '\n')); assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.TRIM_HORIZON); + } + + @Test + public void testWithTimestampAtInitialPositionInStreamVariables() { + KinesisClientLibConfiguration config = + getConfiguration(StringUtils.join(new String[] { + "streamName = a", + "applicationName = b", + "AWSCredentialsProvider = ABCD," + credentialName1, + "timestampAtInitialPositionInStream = 1527267472" + }, '\n')); + + assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.AT_TIMESTAMP); + assertEquals(config.getTimestampAtInitialPositionInStream(), + new Date(1527267472 * 1000L)); + } + + @Test + public void testWithEmptyTimestampAtInitialPositionInStreamVariables() { + KinesisClientLibConfiguration config = + getConfiguration(StringUtils.join(new String[] { + "streamName = a", + "applicationName = b", + "AWSCredentialsProvider = ABCD," + credentialName1, + "timestampAtInitialPositionInStream = " + }, '\n')); + + assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.LATEST); + assertEquals(config.getTimestampAtInitialPositionInStream(), null); + } + + @Test + public void testWithNonNumericTimestampAtInitialPositionInStreamVariables() { + KinesisClientLibConfiguration config = + getConfiguration(StringUtils.join(new String[] { + "streamName = a", + "applicationName = b", + "AWSCredentialsProvider = ABCD," + credentialName1, + "timestampAtInitialPositionInStream = 123abc" + }, '\n')); + + assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.LATEST); + assertEquals(config.getTimestampAtInitialPositionInStream(), null); } @Test @@ -202,6 +267,42 @@ public class KinesisClientLibConfiguratorTest { assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.TRIM_HORIZON); } + @Test + public void testEmptyOptionalVariables() { + KinesisClientLibConfiguration config = + getConfiguration(StringUtils.join(new String[] { + "streamName = a", + "applicationName = b", + "AWSCredentialsProvider = ABCD," + credentialName1, + "workerId = 123", + "initialPositionInStream = TriM_Horizon", + "maxGetRecordsThreadPool = 1" + }, '\n')); + assertEquals(config.getMaxGetRecordsThreadPool(), Optional.of(1)); + assertEquals(config.getRetryGetRecordsInSeconds(), Optional.empty()); + } + + @Test + public void testWithZeroValue() { + String test = StringUtils.join(new String[]{ + "streamName = a", + "applicationName = b", + "AWSCredentialsProvider = ABCD," + credentialName1, + "workerId = 123", + "initialPositionInStream = TriM_Horizon", + "maxGetRecordsThreadPool = 0", + "retryGetRecordsInSeconds = 0" + }, '\n'); + InputStream input = new ByteArrayInputStream(test.getBytes()); + + try { + configurator.getConfiguration(input); + } catch (Exception e) { + fail("Don't expect to fail on invalid variable value"); + + } + } + @Test public void testWithInvalidIntValue() { String test = StringUtils.join(new String[] { diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/CheckpointImplTestBase.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/CheckpointImplTestBase.java index 6e93a296..075a89bd 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/CheckpointImplTestBase.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/CheckpointImplTestBase.java @@ -107,6 +107,68 @@ public abstract class CheckpointImplTestBase { ExtendedSequenceNumber extendedSequenceNumber = new ExtendedSequenceNumber(checkpointValue); checkpoint.setCheckpoint(shardId, new ExtendedSequenceNumber(checkpointValue), concurrencyToken); Assert.assertEquals(extendedSequenceNumber, checkpoint.getCheckpoint(shardId)); + Assert.assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); + Assert.assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); } + @Test + public final void testInitialPrepareCheckpoint() throws Exception { + String sequenceNumber = "1"; + String pendingCheckpointValue = "99999"; + String shardId = "myShardId"; + ExtendedSequenceNumber extendedCheckpointNumber = new ExtendedSequenceNumber(sequenceNumber); + checkpoint.setCheckpoint(shardId, new ExtendedSequenceNumber(sequenceNumber), testConcurrencyToken); + + ExtendedSequenceNumber extendedPendingCheckpointNumber = new ExtendedSequenceNumber(pendingCheckpointValue); + checkpoint.prepareCheckpoint(shardId, new ExtendedSequenceNumber(pendingCheckpointValue), testConcurrencyToken); + + Assert.assertEquals(extendedCheckpointNumber, checkpoint.getCheckpoint(shardId)); + Assert.assertEquals(extendedCheckpointNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); + Assert.assertEquals(extendedPendingCheckpointNumber, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + } + + @Test + public final void testAdvancingPrepareCheckpoint() throws Exception { + String shardId = "myShardId"; + String checkpointValue = "12345"; + ExtendedSequenceNumber extendedCheckpointNumber = new ExtendedSequenceNumber(checkpointValue); + checkpoint.setCheckpoint(shardId, new ExtendedSequenceNumber(checkpointValue), testConcurrencyToken); + + for (Integer i = 0; i < 10; i++) { + String sequenceNumber = i.toString(); + ExtendedSequenceNumber extendedSequenceNumber = new ExtendedSequenceNumber(sequenceNumber); + checkpoint.prepareCheckpoint(shardId, new ExtendedSequenceNumber(sequenceNumber), testConcurrencyToken); + Assert.assertEquals(extendedCheckpointNumber, checkpoint.getCheckpoint(shardId)); + Assert.assertEquals(extendedCheckpointNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); + Assert.assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + } + } + + @Test + public final void testPrepareAndSetCheckpoint() throws Exception { + String checkpointValue = "12345"; + String shardId = "testShardId-1"; + String concurrencyToken = "token-1"; + String pendingCheckpointValue = "99999"; + + // set initial checkpoint + ExtendedSequenceNumber extendedCheckpointNumber = new ExtendedSequenceNumber(checkpointValue); + checkpoint.setCheckpoint(shardId, new ExtendedSequenceNumber(checkpointValue), concurrencyToken); + Assert.assertEquals(extendedCheckpointNumber, checkpoint.getCheckpoint(shardId)); + Assert.assertEquals(extendedCheckpointNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); + Assert.assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + + // prepare checkpoint + ExtendedSequenceNumber extendedPendingCheckpointNumber = new ExtendedSequenceNumber(pendingCheckpointValue); + checkpoint.prepareCheckpoint(shardId, new ExtendedSequenceNumber(pendingCheckpointValue), concurrencyToken); + Assert.assertEquals(extendedCheckpointNumber, checkpoint.getCheckpoint(shardId)); + Assert.assertEquals(extendedCheckpointNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); + Assert.assertEquals(extendedPendingCheckpointNumber, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + + // do checkpoint + checkpoint.setCheckpoint(shardId, new ExtendedSequenceNumber(pendingCheckpointValue), concurrencyToken); + Assert.assertEquals(extendedPendingCheckpointNumber, checkpoint.getCheckpoint(shardId)); + Assert.assertEquals(extendedPendingCheckpointNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); + Assert.assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + } } diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/InMemoryCheckpointImpl.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/InMemoryCheckpointImpl.java index ad761ef5..9cdc31c1 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/InMemoryCheckpointImpl.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/InMemoryCheckpointImpl.java @@ -33,6 +33,7 @@ public class InMemoryCheckpointImpl implements ICheckpoint { private Map checkpoints = new HashMap<>(); private Map flushpoints = new HashMap<>(); + private Map pendingCheckpoints = new HashMap<>(); private final String startingSequenceNumber; /** @@ -95,6 +96,7 @@ public class InMemoryCheckpointImpl implements ICheckpoint { throws KinesisClientLibException { checkpoints.put(shardId, checkpointValue); flushpoints.put(shardId, checkpointValue); + pendingCheckpoints.remove(shardId); if (LOG.isDebugEnabled()) { LOG.debug("shardId: " + shardId + " checkpoint: " + checkpointValue); @@ -112,6 +114,22 @@ public class InMemoryCheckpointImpl implements ICheckpoint { return checkpoint; } + @Override + public void prepareCheckpoint(String shardId, ExtendedSequenceNumber pendingCheckpoint, String concurrencyToken) + throws KinesisClientLibException { + pendingCheckpoints.put(shardId, pendingCheckpoint); + } + + @Override + public Checkpoint getCheckpointObject(String shardId) throws KinesisClientLibException { + ExtendedSequenceNumber checkpoint = flushpoints.get(shardId); + ExtendedSequenceNumber pendingCheckpoint = pendingCheckpoints.get(shardId); + + Checkpoint checkpointObj = new Checkpoint(checkpoint, pendingCheckpoint); + LOG.debug("getCheckpointObject shardId: " + shardId + ", " + checkpointObj); + return checkpointObj; + } + /** Check that string is neither null nor empty. */ static void verifyNotEmpty(String string, String message) { diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/AsynchronousGetRecordsRetrievalStrategyIntegrationTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/AsynchronousGetRecordsRetrievalStrategyIntegrationTest.java new file mode 100644 index 00000000..37f58c1c --- /dev/null +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/AsynchronousGetRecordsRetrievalStrategyIntegrationTest.java @@ -0,0 +1,193 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package com.amazonaws.services.kinesis.clientlibrary.lib.worker; + +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.IsEqual.equalTo; +import static org.junit.Assert.assertNull; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.concurrent.CompletionService; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; + +import com.amazonaws.services.kinesis.model.ExpiredIteratorException; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.runners.MockitoJUnitRunner; + +import com.amazonaws.services.kinesis.clientlibrary.proxies.IKinesisProxy; +import com.amazonaws.services.kinesis.model.GetRecordsResult; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.mockito.stubbing.Answer; + +@RunWith(MockitoJUnitRunner.class) +public class AsynchronousGetRecordsRetrievalStrategyIntegrationTest { + + private static final int CORE_POOL_SIZE = 1; + private static final int MAX_POOL_SIZE = 2; + private static final int TIME_TO_LIVE = 5; + private static final int RETRY_GET_RECORDS_IN_SECONDS = 2; + private static final int SLEEP_GET_RECORDS_IN_SECONDS = 10; + + @Mock + private IKinesisProxy mockKinesisProxy; + @Mock + private ShardInfo mockShardInfo; + @Mock + private Supplier> completionServiceSupplier; + @Mock + private DataFetcherResult result; + @Mock + private GetRecordsResult recordsResult; + + private CompletionService completionService; + + private AsynchronousGetRecordsRetrievalStrategy getRecordsRetrivalStrategy; + private KinesisDataFetcher dataFetcher; + private ExecutorService executorService; + private RejectedExecutionHandler rejectedExecutionHandler; + private int numberOfRecords = 10; + + + @Before + public void setup() { + dataFetcher = spy(new KinesisDataFetcherForTests(mockKinesisProxy, mockShardInfo)); + rejectedExecutionHandler = spy(new ThreadPoolExecutor.AbortPolicy()); + executorService = spy(new ThreadPoolExecutor( + CORE_POOL_SIZE, + MAX_POOL_SIZE, + TIME_TO_LIVE, + TimeUnit.SECONDS, + new LinkedBlockingQueue<>(1), + new ThreadFactoryBuilder().setDaemon(true).setNameFormat("getrecords-worker-%d").build(), + rejectedExecutionHandler)); + completionService = spy(new ExecutorCompletionService(executorService)); + when(completionServiceSupplier.get()).thenReturn(completionService); + getRecordsRetrivalStrategy = new AsynchronousGetRecordsRetrievalStrategy(dataFetcher, executorService, RETRY_GET_RECORDS_IN_SECONDS, completionServiceSupplier, "shardId-0001"); + when(result.accept()).thenReturn(recordsResult); + } + + @Test + public void oneRequestMultithreadTest() { + when(result.accept()).thenReturn(null); + GetRecordsResult getRecordsResult = getRecordsRetrivalStrategy.getRecords(numberOfRecords); + verify(dataFetcher, atLeast(getLeastNumberOfCalls())).getRecords(eq(numberOfRecords)); + verify(executorService, atLeast(getLeastNumberOfCalls())).execute(any()); + assertNull(getRecordsResult); + } + + @Test + public void multiRequestTest() { + ExecutorCompletionService completionService1 = spy(new ExecutorCompletionService(executorService)); + when(completionServiceSupplier.get()).thenReturn(completionService1); + GetRecordsResult getRecordsResult = getRecordsRetrivalStrategy.getRecords(numberOfRecords); + verify(dataFetcher, atLeast(getLeastNumberOfCalls())).getRecords(numberOfRecords); + verify(executorService, atLeast(getLeastNumberOfCalls())).execute(any()); + assertThat(getRecordsResult, equalTo(recordsResult)); + + when(result.accept()).thenReturn(null); + ExecutorCompletionService completionService2 = spy(new ExecutorCompletionService(executorService)); + when(completionServiceSupplier.get()).thenReturn(completionService2); + getRecordsResult = getRecordsRetrivalStrategy.getRecords(numberOfRecords); + assertThat(getRecordsResult, nullValue(GetRecordsResult.class)); + } + + @Test + @Ignore + public void testInterrupted() throws InterruptedException, ExecutionException { + Future mockFuture = mock(Future.class); + when(completionService.submit(any())).thenReturn(mockFuture); + when(completionService.poll()).thenReturn(mockFuture); + doThrow(InterruptedException.class).when(mockFuture).get(); + GetRecordsResult getRecordsResult = getRecordsRetrivalStrategy.getRecords(numberOfRecords); + verify(mockFuture).get(); + assertNull(getRecordsResult); + } + + @Test (expected = ExpiredIteratorException.class) + public void testExpiredIteratorExcpetion() throws InterruptedException { + when(dataFetcher.getRecords(eq(numberOfRecords))).thenAnswer(new Answer() { + @Override + public DataFetcherResult answer(final InvocationOnMock invocationOnMock) throws Throwable { + Thread.sleep(SLEEP_GET_RECORDS_IN_SECONDS * 1000); + throw new ExpiredIteratorException("ExpiredIterator"); + } + }); + + try { + getRecordsRetrivalStrategy.getRecords(numberOfRecords); + } finally { + verify(dataFetcher, atLeast(getLeastNumberOfCalls())).getRecords(eq(numberOfRecords)); + verify(executorService, atLeast(getLeastNumberOfCalls())).execute(any()); + } + } + + private int getLeastNumberOfCalls() { + int leastNumberOfCalls = 0; + for (int i = MAX_POOL_SIZE; i > 0; i--) { + if (i * RETRY_GET_RECORDS_IN_SECONDS <= SLEEP_GET_RECORDS_IN_SECONDS) { + leastNumberOfCalls = i; + break; + } + } + return leastNumberOfCalls; + } + + @After + public void shutdown() { + getRecordsRetrivalStrategy.shutdown(); + verify(executorService).shutdownNow(); + } + + private class KinesisDataFetcherForTests extends KinesisDataFetcher { + public KinesisDataFetcherForTests(final IKinesisProxy kinesisProxy, final ShardInfo shardInfo) { + super(kinesisProxy, shardInfo); + } + + @Override + public DataFetcherResult getRecords(final int maxRecords) { + try { + Thread.sleep(SLEEP_GET_RECORDS_IN_SECONDS * 1000); + } catch (InterruptedException e) { + // Do nothing + } + + return result; + } + } + +} diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/AsynchronousGetRecordsRetrievalStrategyTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/AsynchronousGetRecordsRetrievalStrategyTest.java new file mode 100644 index 00000000..151300de --- /dev/null +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/AsynchronousGetRecordsRetrievalStrategyTest.java @@ -0,0 +1,184 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package com.amazonaws.services.kinesis.clientlibrary.lib.worker; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.concurrent.CompletionService; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; + +import com.amazonaws.services.kinesis.model.ExpiredIteratorException; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; + +import com.amazonaws.services.kinesis.model.GetRecordsResult; + +/** + * + */ +@RunWith(MockitoJUnitRunner.class) +public class AsynchronousGetRecordsRetrievalStrategyTest { + + private static final long RETRY_GET_RECORDS_IN_SECONDS = 5; + private static final String SHARD_ID = "ShardId-0001"; + @Mock + private KinesisDataFetcher dataFetcher; + @Mock + private ExecutorService executorService; + @Mock + private Supplier> completionServiceSupplier; + @Mock + private CompletionService completionService; + @Mock + private Future successfulFuture; + @Mock + private Future blockedFuture; + @Mock + private DataFetcherResult dataFetcherResult; + @Mock + private GetRecordsResult expectedResults; + + @Before + public void before() { + when(completionServiceSupplier.get()).thenReturn(completionService); + when(dataFetcherResult.getResult()).thenReturn(expectedResults); + when(dataFetcherResult.accept()).thenReturn(expectedResults); + } + + @Test + public void testSingleSuccessfulRequestFuture() throws Exception { + AsynchronousGetRecordsRetrievalStrategy strategy = new AsynchronousGetRecordsRetrievalStrategy(dataFetcher, + executorService, (int) RETRY_GET_RECORDS_IN_SECONDS, completionServiceSupplier, SHARD_ID); + + when(executorService.isShutdown()).thenReturn(false); + when(completionService.submit(any())).thenReturn(successfulFuture); + when(completionService.poll(anyLong(), any())).thenReturn(successfulFuture); + when(successfulFuture.get()).thenReturn(dataFetcherResult); + + GetRecordsResult result = strategy.getRecords(10); + + verify(executorService).isShutdown(); + verify(completionService).submit(any()); + verify(completionService).poll(eq(RETRY_GET_RECORDS_IN_SECONDS), eq(TimeUnit.SECONDS)); + verify(successfulFuture).get(); + verify(successfulFuture).cancel(eq(true)); + + assertThat(result, equalTo(expectedResults)); + } + + @Test + public void testBlockedAndSuccessfulFuture() throws Exception { + AsynchronousGetRecordsRetrievalStrategy strategy = new AsynchronousGetRecordsRetrievalStrategy(dataFetcher, + executorService, (int) RETRY_GET_RECORDS_IN_SECONDS, completionServiceSupplier, SHARD_ID); + + when(executorService.isShutdown()).thenReturn(false); + when(completionService.submit(any())).thenReturn(blockedFuture).thenReturn(successfulFuture); + when(completionService.poll(anyLong(), any())).thenReturn(null).thenReturn(successfulFuture); + when(successfulFuture.get()).thenReturn(dataFetcherResult); + when(successfulFuture.cancel(anyBoolean())).thenReturn(false); + when(blockedFuture.cancel(anyBoolean())).thenReturn(true); + when(successfulFuture.isCancelled()).thenReturn(false); + when(blockedFuture.isCancelled()).thenReturn(true); + + GetRecordsResult actualResults = strategy.getRecords(10); + + verify(completionService, times(2)).submit(any()); + verify(completionService, times(2)).poll(eq(RETRY_GET_RECORDS_IN_SECONDS), eq(TimeUnit.SECONDS)); + verify(successfulFuture).get(); + verify(blockedFuture, never()).get(); + verify(successfulFuture).cancel(eq(true)); + verify(blockedFuture).cancel(eq(true)); + + assertThat(actualResults, equalTo(expectedResults)); + } + + @Test(expected = IllegalStateException.class) + public void testStrategyIsShutdown() throws Exception { + AsynchronousGetRecordsRetrievalStrategy strategy = new AsynchronousGetRecordsRetrievalStrategy(dataFetcher, + executorService, (int) RETRY_GET_RECORDS_IN_SECONDS, completionServiceSupplier, SHARD_ID); + + when(executorService.isShutdown()).thenReturn(true); + + strategy.getRecords(10); + } + + @Test + public void testPoolOutOfResources() throws Exception { + AsynchronousGetRecordsRetrievalStrategy strategy = new AsynchronousGetRecordsRetrievalStrategy(dataFetcher, + executorService, (int) RETRY_GET_RECORDS_IN_SECONDS, completionServiceSupplier, SHARD_ID); + + when(executorService.isShutdown()).thenReturn(false); + when(completionService.submit(any())).thenReturn(blockedFuture).thenThrow(new RejectedExecutionException("Rejected!")).thenReturn(successfulFuture); + when(completionService.poll(anyLong(), any())).thenReturn(null).thenReturn(null).thenReturn(successfulFuture); + when(successfulFuture.get()).thenReturn(dataFetcherResult); + when(successfulFuture.cancel(anyBoolean())).thenReturn(false); + when(blockedFuture.cancel(anyBoolean())).thenReturn(true); + when(successfulFuture.isCancelled()).thenReturn(false); + when(blockedFuture.isCancelled()).thenReturn(true); + + GetRecordsResult actualResult = strategy.getRecords(10); + + verify(completionService, times(3)).submit(any()); + verify(completionService, times(3)).poll(eq(RETRY_GET_RECORDS_IN_SECONDS), eq(TimeUnit.SECONDS)); + verify(successfulFuture).cancel(eq(true)); + verify(blockedFuture).cancel(eq(true)); + + + assertThat(actualResult, equalTo(expectedResults)); + } + + @Test (expected = ExpiredIteratorException.class) + public void testExpiredIteratorExceptionCase() throws Exception { + AsynchronousGetRecordsRetrievalStrategy strategy = new AsynchronousGetRecordsRetrievalStrategy(dataFetcher, + executorService, (int) RETRY_GET_RECORDS_IN_SECONDS, completionServiceSupplier, SHARD_ID); + Future successfulFuture2 = mock(Future.class); + + when(executorService.isShutdown()).thenReturn(false); + when(completionService.submit(any())).thenReturn(successfulFuture, successfulFuture2); + when(completionService.poll(anyLong(), any())).thenReturn(null).thenReturn(successfulFuture); + when(successfulFuture.get()).thenThrow(new ExecutionException(new ExpiredIteratorException("ExpiredException"))); + + try { + strategy.getRecords(10); + } finally { + verify(executorService).isShutdown(); + verify(completionService, times(2)).submit(any()); + verify(completionService, times(2)).poll(eq(RETRY_GET_RECORDS_IN_SECONDS), eq(TimeUnit.SECONDS)); + verify(successfulFuture).cancel(eq(true)); + verify(successfulFuture2).cancel(eq(true)); + } + } + +} diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/BlockingGetRecordsCacheTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/BlockingGetRecordsCacheTest.java new file mode 100644 index 00000000..0636baea --- /dev/null +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/BlockingGetRecordsCacheTest.java @@ -0,0 +1,83 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazonaws.services.kinesis.clientlibrary.lib.worker; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.when; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; + +import com.amazonaws.services.kinesis.clientlibrary.types.ProcessRecordsInput; +import com.amazonaws.services.kinesis.model.GetRecordsResult; +import com.amazonaws.services.kinesis.model.Record; + +/** + * Test class for the BlockingGetRecordsCache class. + */ +@RunWith(MockitoJUnitRunner.class) +public class BlockingGetRecordsCacheTest { + private static final int MAX_RECORDS_PER_COUNT = 10_000; + + @Mock + private GetRecordsRetrievalStrategy getRecordsRetrievalStrategy; + @Mock + private GetRecordsResult getRecordsResult; + + private List records; + private BlockingGetRecordsCache blockingGetRecordsCache; + + @Before + public void setup() { + records = new ArrayList<>(); + blockingGetRecordsCache = new BlockingGetRecordsCache(MAX_RECORDS_PER_COUNT, getRecordsRetrievalStrategy); + + when(getRecordsRetrievalStrategy.getRecords(eq(MAX_RECORDS_PER_COUNT))).thenReturn(getRecordsResult); + when(getRecordsResult.getRecords()).thenReturn(records); + } + + @Test + public void testGetNextRecordsWithNoRecords() { + ProcessRecordsInput result = blockingGetRecordsCache.getNextResult(); + + assertEquals(result.getRecords(), records); + assertNull(result.getCacheEntryTime()); + assertNull(result.getCacheExitTime()); + assertEquals(result.getTimeSpentInCache(), Duration.ZERO); + } + + @Test + public void testGetNextRecordsWithRecords() { + Record record = new Record(); + records.add(record); + records.add(record); + records.add(record); + records.add(record); + + ProcessRecordsInput result = blockingGetRecordsCache.getNextResult(); + + assertEquals(result.getRecords(), records); + } +} diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ConsumerStatesTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ConsumerStatesTest.java index c0a778e9..fa163ad2 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ConsumerStatesTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ConsumerStatesTest.java @@ -1,3 +1,17 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; import static com.amazonaws.services.kinesis.clientlibrary.lib.worker.ConsumerStates.ConsumerState; @@ -41,6 +55,8 @@ public class ConsumerStatesTest { @Mock private IRecordProcessor recordProcessor; @Mock + private KinesisClientLibConfiguration config; + @Mock private RecordProcessorCheckpointer recordProcessorCheckpointer; @Mock private ExecutorService executorService; @@ -60,6 +76,8 @@ public class ConsumerStatesTest { private IKinesisProxy kinesisProxy; @Mock private InitialPositionInStreamExtended initialPositionInStream; + @Mock + private GetRecordsCache getRecordsCache; private long parentShardPollIntervalMillis = 0xCAFE; private boolean cleanupLeasesOfCompletedShards = true; @@ -82,7 +100,7 @@ public class ConsumerStatesTest { when(consumer.isCleanupLeasesOfCompletedShards()).thenReturn(cleanupLeasesOfCompletedShards); when(consumer.getTaskBackoffTimeMillis()).thenReturn(taskBackoffTimeMillis); when(consumer.getShutdownReason()).thenReturn(reason); - + when(consumer.getGetRecordsCache()).thenReturn(getRecordsCache); } private static final Class> LEASE_MANAGER_CLASS = (Class>) (Class) ILeaseManager.class; @@ -138,7 +156,7 @@ public class ConsumerStatesTest { } @Test - public void processingStateTest() { + public void processingStateTestSynchronous() { ConsumerState state = ShardConsumerState.PROCESSING.getConsumerState(); ITask task = state.createTask(consumer); @@ -164,6 +182,60 @@ public class ConsumerStatesTest { } + @Test + public void processingStateTestAsynchronous() { + ConsumerState state = ShardConsumerState.PROCESSING.getConsumerState(); + ITask task = state.createTask(consumer); + + assertThat(task, procTask(ShardInfo.class, "shardInfo", equalTo(shardInfo))); + assertThat(task, procTask(IRecordProcessor.class, "recordProcessor", equalTo(recordProcessor))); + assertThat(task, procTask(RecordProcessorCheckpointer.class, "recordProcessorCheckpointer", + equalTo(recordProcessorCheckpointer))); + assertThat(task, procTask(KinesisDataFetcher.class, "dataFetcher", equalTo(dataFetcher))); + assertThat(task, procTask(StreamConfig.class, "streamConfig", equalTo(streamConfig))); + assertThat(task, procTask(Long.class, "backoffTimeMillis", equalTo(taskBackoffTimeMillis))); + + assertThat(state.successTransition(), equalTo(ShardConsumerState.PROCESSING.getConsumerState())); + + assertThat(state.shutdownTransition(ShutdownReason.ZOMBIE), + equalTo(ShardConsumerState.SHUTTING_DOWN.getConsumerState())); + assertThat(state.shutdownTransition(ShutdownReason.TERMINATE), + equalTo(ShardConsumerState.SHUTTING_DOWN.getConsumerState())); + assertThat(state.shutdownTransition(ShutdownReason.REQUESTED), + equalTo(ShardConsumerState.SHUTDOWN_REQUESTED.getConsumerState())); + + assertThat(state.getState(), equalTo(ShardConsumerState.PROCESSING)); + assertThat(state.getTaskType(), equalTo(TaskType.PROCESS)); + + } + + @Test + public void processingStateRecordsFetcher() { + + ConsumerState state = ShardConsumerState.PROCESSING.getConsumerState(); + ITask task = state.createTask(consumer); + + assertThat(task, procTask(ShardInfo.class, "shardInfo", equalTo(shardInfo))); + assertThat(task, procTask(IRecordProcessor.class, "recordProcessor", equalTo(recordProcessor))); + assertThat(task, procTask(RecordProcessorCheckpointer.class, "recordProcessorCheckpointer", + equalTo(recordProcessorCheckpointer))); + assertThat(task, procTask(KinesisDataFetcher.class, "dataFetcher", equalTo(dataFetcher))); + assertThat(task, procTask(StreamConfig.class, "streamConfig", equalTo(streamConfig))); + assertThat(task, procTask(Long.class, "backoffTimeMillis", equalTo(taskBackoffTimeMillis))); + + assertThat(state.successTransition(), equalTo(ShardConsumerState.PROCESSING.getConsumerState())); + + assertThat(state.shutdownTransition(ShutdownReason.ZOMBIE), + equalTo(ShardConsumerState.SHUTTING_DOWN.getConsumerState())); + assertThat(state.shutdownTransition(ShutdownReason.TERMINATE), + equalTo(ShardConsumerState.SHUTTING_DOWN.getConsumerState())); + assertThat(state.shutdownTransition(ShutdownReason.REQUESTED), + equalTo(ShardConsumerState.SHUTDOWN_REQUESTED.getConsumerState())); + + assertThat(state.getState(), equalTo(ShardConsumerState.PROCESSING)); + assertThat(state.getTaskType(), equalTo(TaskType.PROCESS)); + } + @Test public void shutdownRequestState() { ConsumerState state = ShardConsumerState.SHUTDOWN_REQUESTED.getConsumerState(); @@ -270,7 +342,7 @@ public class ConsumerStatesTest { } static ReflectionPropertyMatcher shutdownTask(Class valueTypeClass, - String propertyName, Matcher matcher) { + String propertyName, Matcher matcher) { return taskWith(ShutdownTask.class, valueTypeClass, propertyName, matcher); } @@ -280,17 +352,17 @@ public class ConsumerStatesTest { } static ReflectionPropertyMatcher procTask(Class valueTypeClass, - String propertyName, Matcher matcher) { + String propertyName, Matcher matcher) { return taskWith(ProcessTask.class, valueTypeClass, propertyName, matcher); } static ReflectionPropertyMatcher initTask(Class valueTypeClass, - String propertyName, Matcher matcher) { + String propertyName, Matcher matcher) { return taskWith(InitializeTask.class, valueTypeClass, propertyName, matcher); } static ReflectionPropertyMatcher taskWith(Class taskTypeClass, - Class valueTypeClass, String propertyName, Matcher matcher) { + Class valueTypeClass, String propertyName, Matcher matcher) { return new ReflectionPropertyMatcher<>(taskTypeClass, valueTypeClass, matcher, propertyName); } @@ -303,7 +375,7 @@ public class ConsumerStatesTest { private final Field matchingField; private ReflectionPropertyMatcher(Class taskTypeClass, Class valueTypeClass, - Matcher matcher, String propertyName) { + Matcher matcher, String propertyName) { this.taskTypeClass = taskTypeClass; this.valueTypeClazz = valueTypeClass; this.matcher = matcher; diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/GracefulShutdownCoordinatorTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/GracefulShutdownCoordinatorTest.java new file mode 100644 index 00000000..c032bf0c --- /dev/null +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/GracefulShutdownCoordinatorTest.java @@ -0,0 +1,322 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package com.amazonaws.services.kinesis.clientlibrary.lib.worker; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.not; +import static org.junit.Assert.assertThat; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; +import org.mockito.verification.VerificationMode; + +@RunWith(MockitoJUnitRunner.class) +public class GracefulShutdownCoordinatorTest { + + @Mock + private CountDownLatch shutdownCompleteLatch; + @Mock + private CountDownLatch notificationCompleteLatch; + @Mock + private Worker worker; + @Mock + private Callable contextCallable; + @Mock + private ConcurrentMap shardInfoConsumerMap; + + @Test + public void testAllShutdownCompletedAlready() throws Exception { + Callable requestedShutdownCallable = buildRequestedShutdownCallable(); + + when(shutdownCompleteLatch.await(anyLong(), any(TimeUnit.class))).thenReturn(true); + when(notificationCompleteLatch.await(anyLong(), any(TimeUnit.class))).thenReturn(true); + + assertThat(requestedShutdownCallable.call(), equalTo(true)); + verify(shutdownCompleteLatch).await(anyLong(), any(TimeUnit.class)); + verify(notificationCompleteLatch).await(anyLong(), any(TimeUnit.class)); + verify(worker).shutdown(); + } + + @Test + public void testNotificationNotCompletedYet() throws Exception { + Callable requestedShutdownCallable = buildRequestedShutdownCallable(); + + mockLatchAwait(notificationCompleteLatch, false, true); + when(notificationCompleteLatch.getCount()).thenReturn(1L, 0L); + mockLatchAwait(shutdownCompleteLatch, true); + when(shutdownCompleteLatch.getCount()).thenReturn(1L, 1L, 0L); + + when(worker.isShutdownComplete()).thenReturn(false, true); + mockShardInfoConsumerMap(1, 0); + + assertThat(requestedShutdownCallable.call(), equalTo(true)); + verify(notificationCompleteLatch, times(2)).await(anyLong(), any(TimeUnit.class)); + verify(notificationCompleteLatch).getCount(); + + verify(shutdownCompleteLatch).await(anyLong(), any(TimeUnit.class)); + verify(shutdownCompleteLatch, times(2)).getCount(); + + verify(worker).shutdown(); + } + + @Test + public void testShutdownNotCompletedYet() throws Exception { + Callable requestedShutdownCallable = buildRequestedShutdownCallable(); + + mockLatchAwait(notificationCompleteLatch, true); + mockLatchAwait(shutdownCompleteLatch, false, true); + when(shutdownCompleteLatch.getCount()).thenReturn(1L, 0L); + + when(worker.isShutdownComplete()).thenReturn(false, true); + mockShardInfoConsumerMap(1, 0); + + assertThat(requestedShutdownCallable.call(), equalTo(true)); + verify(notificationCompleteLatch).await(anyLong(), any(TimeUnit.class)); + verify(notificationCompleteLatch, never()).getCount(); + + verify(shutdownCompleteLatch, times(2)).await(anyLong(), any(TimeUnit.class)); + verify(shutdownCompleteLatch, times(2)).getCount(); + + verify(worker).shutdown(); + } + + @Test + public void testMultipleAttemptsForNotification() throws Exception { + Callable requestedShutdownCallable = buildRequestedShutdownCallable(); + + mockLatchAwait(notificationCompleteLatch, false, false, true); + when(notificationCompleteLatch.getCount()).thenReturn(2L, 1L, 0L); + + mockLatchAwait(shutdownCompleteLatch, true); + when(shutdownCompleteLatch.getCount()).thenReturn(2L, 2L, 1L, 1L, 0L); + + when(worker.isShutdownComplete()).thenReturn(false, false, false, true); + mockShardInfoConsumerMap(2, 1, 0); + + assertThat(requestedShutdownCallable.call(), equalTo(true)); + + verifyLatchAwait(notificationCompleteLatch, 3); + verify(notificationCompleteLatch, times(2)).getCount(); + + verifyLatchAwait(shutdownCompleteLatch, 1); + verify(shutdownCompleteLatch, times(4)).getCount(); + } + + @Test + public void testWorkerAlreadyShutdownAtNotification() throws Exception { + Callable requestedShutdownCallable = buildRequestedShutdownCallable(); + + mockLatchAwait(notificationCompleteLatch, false, true); + when(notificationCompleteLatch.getCount()).thenReturn(1L, 0L); + + mockLatchAwait(shutdownCompleteLatch, true); + when(shutdownCompleteLatch.getCount()).thenReturn(1L, 1L, 0L); + + when(worker.isShutdownComplete()).thenReturn(true); + mockShardInfoConsumerMap(0); + + assertThat(requestedShutdownCallable.call(), equalTo(false)); + + verifyLatchAwait(notificationCompleteLatch); + verify(notificationCompleteLatch).getCount(); + + verifyLatchAwait(shutdownCompleteLatch, never()); + verify(shutdownCompleteLatch, times(3)).getCount(); + } + + @Test + public void testWorkerAlreadyShutdownAtComplete() throws Exception { + Callable requestedShutdownCallable = buildRequestedShutdownCallable(); + + mockLatchAwait(notificationCompleteLatch, true); + + mockLatchAwait(shutdownCompleteLatch, false, true); + when(shutdownCompleteLatch.getCount()).thenReturn(1L, 1L, 1L); + + when(worker.isShutdownComplete()).thenReturn(true); + mockShardInfoConsumerMap(0); + + assertThat(requestedShutdownCallable.call(), equalTo(false)); + + verifyLatchAwait(notificationCompleteLatch); + verify(notificationCompleteLatch, never()).getCount(); + + verifyLatchAwait(shutdownCompleteLatch); + verify(shutdownCompleteLatch, times(3)).getCount(); + } + + @Test + public void testNotificationInterrupted() throws Exception { + Callable requestedShutdownCallable = buildRequestedShutdownCallable(); + + when(notificationCompleteLatch.await(anyLong(), any(TimeUnit.class))).thenThrow(new InterruptedException()); + when(notificationCompleteLatch.getCount()).thenReturn(1L); + + when(shutdownCompleteLatch.getCount()).thenReturn(1L); + + assertThat(requestedShutdownCallable.call(), equalTo(false)); + verifyLatchAwait(notificationCompleteLatch); + verifyLatchAwait(shutdownCompleteLatch, never()); + verify(worker, never()).shutdown(); + } + + @Test + public void testShutdownInterrupted() throws Exception { + Callable requestedShutdownCallable = buildRequestedShutdownCallable(); + + when(notificationCompleteLatch.await(anyLong(), any(TimeUnit.class))).thenReturn(true); + + when(shutdownCompleteLatch.await(anyLong(), any(TimeUnit.class))).thenThrow(new InterruptedException()); + when(shutdownCompleteLatch.getCount()).thenReturn(1L); + + assertThat(requestedShutdownCallable.call(), equalTo(false)); + verifyLatchAwait(notificationCompleteLatch); + verifyLatchAwait(shutdownCompleteLatch); + verify(worker).shutdown(); + } + + @Test + public void testInterruptedAfterNotification() throws Exception { + Callable requestedShutdownCallable = buildRequestedShutdownCallable(); + + when(notificationCompleteLatch.await(anyLong(), any(TimeUnit.class))).thenAnswer(invocation -> { + Thread.currentThread().interrupt(); + return true; + }); + + assertThat(requestedShutdownCallable.call(), equalTo(false)); + verifyLatchAwait(notificationCompleteLatch); + verifyLatchAwait(shutdownCompleteLatch, never()); + verify(worker, never()).shutdown(); + } + + @Test + public void testInterruptedAfterWorkerShutdown() throws Exception { + Callable requestedShutdownCallable = buildRequestedShutdownCallable(); + + when(notificationCompleteLatch.await(anyLong(), any(TimeUnit.class))).thenReturn(true); + + doAnswer(invocation -> { + Thread.currentThread().interrupt(); + return true; + }).when(worker).shutdown(); + + assertThat(requestedShutdownCallable.call(), equalTo(false)); + verifyLatchAwait(notificationCompleteLatch); + verifyLatchAwait(shutdownCompleteLatch, never()); + verify(worker).shutdown(); + } + + @Test + public void testInterruptedDuringNotification() throws Exception { + Callable requestedShutdownCallable = buildRequestedShutdownCallable(); + + when(notificationCompleteLatch.await(anyLong(), any(TimeUnit.class))).thenAnswer(invocation -> { + Thread.currentThread().interrupt(); + return false; + }); + when(notificationCompleteLatch.getCount()).thenReturn(1L); + + when(shutdownCompleteLatch.getCount()).thenReturn(1L); + + assertThat(requestedShutdownCallable.call(), equalTo(false)); + verifyLatchAwait(notificationCompleteLatch); + verify(notificationCompleteLatch).getCount(); + + verifyLatchAwait(shutdownCompleteLatch, never()); + verify(shutdownCompleteLatch).getCount(); + + verify(worker, never()).shutdown(); + } + + @Test + public void testInterruptedDuringShutdown() throws Exception { + Callable requestedShutdownCallable = buildRequestedShutdownCallable(); + + when(notificationCompleteLatch.await(anyLong(), any(TimeUnit.class))).thenReturn(true); + + when(shutdownCompleteLatch.await(anyLong(), any(TimeUnit.class))).thenAnswer(invocation -> { + Thread.currentThread().interrupt(); + return false; + }); + when(shutdownCompleteLatch.getCount()).thenReturn(1L); + + assertThat(requestedShutdownCallable.call(), equalTo(false)); + verifyLatchAwait(notificationCompleteLatch); + verify(notificationCompleteLatch, never()).getCount(); + + verifyLatchAwait(shutdownCompleteLatch); + verify(shutdownCompleteLatch).getCount(); + + verify(worker).shutdown(); + } + + @Test(expected = IllegalStateException.class) + public void testWorkerShutdownCallableThrows() throws Exception { + Callable requestedShutdownCallable = new GracefulShutdownCoordinator().createGracefulShutdownCallable(contextCallable); + when(contextCallable.call()).thenThrow(new IllegalStateException("Bad Shutdown")); + + requestedShutdownCallable.call(); + } + + private void verifyLatchAwait(CountDownLatch latch) throws Exception { + verifyLatchAwait(latch, times(1)); + } + + private void verifyLatchAwait(CountDownLatch latch, int times) throws Exception { + verifyLatchAwait(latch, times(times)); + } + + private void verifyLatchAwait(CountDownLatch latch, VerificationMode verificationMode) throws Exception { + verify(latch, verificationMode).await(anyLong(), any(TimeUnit.class)); + } + + private void mockLatchAwait(CountDownLatch latch, Boolean initial, Boolean... remaining) throws Exception { + when(latch.await(anyLong(), any(TimeUnit.class))).thenReturn(initial, remaining); + } + + private Callable buildRequestedShutdownCallable() throws Exception { + GracefulShutdownContext context = new GracefulShutdownContext(shutdownCompleteLatch, + notificationCompleteLatch, worker); + when(contextCallable.call()).thenReturn(context); + return new GracefulShutdownCoordinator().createGracefulShutdownCallable(contextCallable); + } + + private void mockShardInfoConsumerMap(Integer initialItemCount, Integer... additionalItemCounts) { + when(worker.getShardInfoShardConsumerMap()).thenReturn(shardInfoConsumerMap); + Boolean additionalEmptyStates[] = new Boolean[additionalItemCounts.length]; + for (int i = 0; i < additionalItemCounts.length; ++i) { + additionalEmptyStates[i] = additionalItemCounts[i] == 0; + } + when(shardInfoConsumerMap.size()).thenReturn(initialItemCount, additionalItemCounts); + when(shardInfoConsumerMap.isEmpty()).thenReturn(initialItemCount == 0, additionalEmptyStates); + } + +} \ No newline at end of file diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfigurationTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfigurationTest.java index 4874a164..7184c9a3 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfigurationTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfigurationTest.java @@ -1,25 +1,26 @@ /* - * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.fail; -import junit.framework.Assert; +import java.util.Date; import org.junit.Test; import org.mockito.Mockito; @@ -35,7 +36,7 @@ import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorF import com.amazonaws.services.kinesis.metrics.interfaces.MetricsLevel; import com.google.common.collect.ImmutableSet; -import java.util.Date; +import junit.framework.Assert; public class KinesisClientLibConfigurationTest { private static final long INVALID_LONG = 0L; @@ -84,7 +85,8 @@ public class KinesisClientLibConfigurationTest { TEST_VALUE_LONG, TEST_VALUE_INT, skipCheckpointValidationValue, - null); + null, + TEST_VALUE_LONG); } @Test @@ -94,7 +96,8 @@ public class KinesisClientLibConfigurationTest { // Try each argument at one time. KinesisClientLibConfiguration config = null; long[] longValues = - { TEST_VALUE_LONG, TEST_VALUE_LONG, TEST_VALUE_LONG, TEST_VALUE_LONG, TEST_VALUE_LONG, TEST_VALUE_LONG }; + { TEST_VALUE_LONG, TEST_VALUE_LONG, TEST_VALUE_LONG, TEST_VALUE_LONG, TEST_VALUE_LONG, TEST_VALUE_LONG, + TEST_VALUE_LONG }; for (int i = 0; i < PARAMETER_COUNT; i++) { longValues[i] = INVALID_LONG; try { @@ -122,7 +125,8 @@ public class KinesisClientLibConfigurationTest { longValues[5], TEST_VALUE_INT, skipCheckpointValidationValue, - null); + null, + longValues[6]); } catch (IllegalArgumentException e) { System.out.println(e.getMessage()); } @@ -156,7 +160,8 @@ public class KinesisClientLibConfigurationTest { TEST_VALUE_LONG, intValues[1], skipCheckpointValidationValue, - null); + null, + TEST_VALUE_LONG); } catch (IllegalArgumentException e) { System.out.println(e.getMessage()); } @@ -263,8 +268,8 @@ public class KinesisClientLibConfigurationTest { IRecordProcessorFactory processorFactory = Mockito.mock(IRecordProcessorFactory.class); new Worker(processorFactory, kclConfig); - Mockito.verify(kclConfig, Mockito.times(9)).getRegionName(); - Mockito.verify(kclConfig, Mockito.times(4)).getKinesisEndpoint(); + Mockito.verify(kclConfig, Mockito.times(5)).getRegionName(); + Mockito.verify(kclConfig, Mockito.times(2)).getKinesisEndpoint(); kclConfig = Mockito.spy( new KinesisClientLibConfiguration("Test", "Test", credentialsProvider, "0") @@ -272,59 +277,11 @@ public class KinesisClientLibConfigurationTest { new Worker(processorFactory, kclConfig); - Mockito.verify(kclConfig, Mockito.times(3)).getRegionName(); - Mockito.verify(kclConfig, Mockito.times(3)).getKinesisEndpoint(); + Mockito.verify(kclConfig, Mockito.times(2)).getRegionName(); + Mockito.verify(kclConfig, Mockito.times(2)).getKinesisEndpoint(); } - @Test - public void testKCLConfigurationWithMultiRegionWithIlligalRegionName() { - // test with illegal region name - AWSCredentialsProvider credentialsProvider = Mockito.mock(AWSCredentialsProvider.class); - KinesisClientLibConfiguration kclConfig = - new KinesisClientLibConfiguration("Test", "Test", credentialsProvider, "0"); - try { - kclConfig = kclConfig.withRegionName("abcd"); - Assert.fail("No expected Exception is thrown."); - } catch (IllegalArgumentException e) { - System.out.println(e.getMessage()); - } - } - - @Test - public void testKCLConfigurationWithMultiRegionWithIlligalRegionNameInFullConstructor() { - // test with illegal region name - Mockito.mock(AWSCredentialsProvider.class); - try { - new KinesisClientLibConfiguration(TEST_STRING, - TEST_STRING, - TEST_STRING, - TEST_STRING, - null, - null, - null, - null, - TEST_VALUE_LONG, - TEST_STRING, - 3, - TEST_VALUE_LONG, - false, - TEST_VALUE_LONG, - TEST_VALUE_LONG, - true, - new ClientConfiguration(), - new ClientConfiguration(), - new ClientConfiguration(), - TEST_VALUE_LONG, - TEST_VALUE_LONG, - 1, - skipCheckpointValidationValue, - "abcd"); - Assert.fail("No expected Exception is thrown."); - } catch(IllegalArgumentException e) { - System.out.println(e.getMessage()); - } - } @Test public void testKCLConfigurationMetricsDefaults() { @@ -402,4 +359,14 @@ public class KinesisClientLibConfigurationTest { fail("Should not have thrown"); } } + + @Test + public void testKCLConfigurationWithIgnoreUnexpectedChildShards() { + KinesisClientLibConfiguration config = + new KinesisClientLibConfiguration("TestApplication", "TestStream", null, "TestWorker"); + // By default, unexpected child shards should not be ignored. + assertFalse(config.shouldIgnoreUnexpectedChildShards()); + config = config.withIgnoreUnexpectedChildShards(true); + assertTrue(config.shouldIgnoreUnexpectedChildShards()); + } } diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisDataFetcherTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisDataFetcherTest.java index dd56a256..fbe720ae 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisDataFetcherTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisDataFetcherTest.java @@ -1,36 +1,48 @@ /* - * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.collection.IsEmptyCollection.empty; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.util.ArrayList; +import java.util.Collections; import java.util.Date; import java.util.List; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; -import com.amazonaws.services.kinesis.model.GetRecordsResult; -import com.amazonaws.services.kinesis.model.Record; -import com.amazonaws.services.kinesis.model.ResourceNotFoundException; -import com.amazonaws.services.kinesis.model.ShardIteratorType; import com.amazonaws.services.kinesis.clientlibrary.exceptions.KinesisClientLibException; import com.amazonaws.services.kinesis.clientlibrary.interfaces.ICheckpoint; import com.amazonaws.services.kinesis.clientlibrary.lib.checkpoint.SentinelCheckpoint; @@ -39,12 +51,20 @@ import com.amazonaws.services.kinesis.clientlibrary.proxies.KinesisProxy; import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber; import com.amazonaws.services.kinesis.metrics.impl.MetricsHelper; import com.amazonaws.services.kinesis.metrics.impl.NullMetricsFactory; +import com.amazonaws.services.kinesis.model.GetRecordsResult; +import com.amazonaws.services.kinesis.model.Record; +import com.amazonaws.services.kinesis.model.ResourceNotFoundException; +import com.amazonaws.services.kinesis.model.ShardIteratorType; /** * Unit tests for KinesisDataFetcher. */ +@RunWith(MockitoJUnitRunner.class) public class KinesisDataFetcherTest { + @Mock + private KinesisProxy kinesisProxy; + private static final int MAX_RECORDS = 1; private static final String SHARD_ID = "shardId-1"; private static final String AT_SEQUENCE_NUMBER = ShardIteratorType.AT_SEQUENCE_NUMBER.toString(); @@ -117,6 +137,7 @@ public class KinesisDataFetcherTest { ICheckpoint checkpoint = mock(ICheckpoint.class); KinesisDataFetcher fetcher = new KinesisDataFetcher(kinesis, SHARD_INFO); + GetRecordsRetrievalStrategy getRecordsRetrievalStrategy = new SynchronousGetRecordsRetrievalStrategy(fetcher); String iteratorA = "foo"; String iteratorB = "bar"; @@ -138,10 +159,10 @@ public class KinesisDataFetcherTest { fetcher.initialize(seqA, null); fetcher.advanceIteratorTo(seqA, null); - Assert.assertEquals(recordsA, fetcher.getRecords(MAX_RECORDS).getRecords()); + Assert.assertEquals(recordsA, getRecordsRetrievalStrategy.getRecords(MAX_RECORDS).getRecords()); fetcher.advanceIteratorTo(seqB, null); - Assert.assertEquals(recordsB, fetcher.getRecords(MAX_RECORDS).getRecords()); + Assert.assertEquals(recordsB, getRecordsRetrievalStrategy.getRecords(MAX_RECORDS).getRecords()); } @Test @@ -181,12 +202,149 @@ public class KinesisDataFetcherTest { // Create data fectcher and initialize it with latest type checkpoint KinesisDataFetcher dataFetcher = new KinesisDataFetcher(mockProxy, SHARD_INFO); dataFetcher.initialize(SentinelCheckpoint.LATEST.toString(), INITIAL_POSITION_LATEST); + GetRecordsRetrievalStrategy getRecordsRetrievalStrategy = new SynchronousGetRecordsRetrievalStrategy(dataFetcher); // Call getRecords of dataFetcher which will throw an exception - dataFetcher.getRecords(maxRecords); + getRecordsRetrievalStrategy.getRecords(maxRecords); // Test shard has reached the end Assert.assertTrue("Shard should reach the end", dataFetcher.isShardEndReached()); } + + @Test + public void testNonNullGetRecords() { + String nextIterator = "TestIterator"; + int maxRecords = 100; + + KinesisProxy mockProxy = mock(KinesisProxy.class); + doThrow(new ResourceNotFoundException("Test Exception")).when(mockProxy).get(nextIterator, maxRecords); + + KinesisDataFetcher dataFetcher = new KinesisDataFetcher(mockProxy, SHARD_INFO); + dataFetcher.initialize(SentinelCheckpoint.LATEST.toString(), INITIAL_POSITION_LATEST); + + DataFetcherResult dataFetcherResult = dataFetcher.getRecords(maxRecords); + + assertThat(dataFetcherResult, notNullValue()); + } + + @Test + public void testFetcherDoesNotAdvanceWithoutAccept() { + final String INITIAL_ITERATOR = "InitialIterator"; + final String NEXT_ITERATOR_ONE = "NextIteratorOne"; + final String NEXT_ITERATOR_TWO = "NextIteratorTwo"; + when(kinesisProxy.getIterator(anyString(), anyString())).thenReturn(INITIAL_ITERATOR); + GetRecordsResult iteratorOneResults = mock(GetRecordsResult.class); + when(iteratorOneResults.getNextShardIterator()).thenReturn(NEXT_ITERATOR_ONE); + when(kinesisProxy.get(eq(INITIAL_ITERATOR), anyInt())).thenReturn(iteratorOneResults); + + GetRecordsResult iteratorTwoResults = mock(GetRecordsResult.class); + when(kinesisProxy.get(eq(NEXT_ITERATOR_ONE), anyInt())).thenReturn(iteratorTwoResults); + when(iteratorTwoResults.getNextShardIterator()).thenReturn(NEXT_ITERATOR_TWO); + + GetRecordsResult finalResult = mock(GetRecordsResult.class); + when(kinesisProxy.get(eq(NEXT_ITERATOR_TWO), anyInt())).thenReturn(finalResult); + when(finalResult.getNextShardIterator()).thenReturn(null); + + KinesisDataFetcher dataFetcher = new KinesisDataFetcher(kinesisProxy, SHARD_INFO); + dataFetcher.initialize("TRIM_HORIZON", + InitialPositionInStreamExtended.newInitialPosition(InitialPositionInStream.TRIM_HORIZON)); + + assertNoAdvance(dataFetcher, iteratorOneResults, INITIAL_ITERATOR); + assertAdvanced(dataFetcher, iteratorOneResults, INITIAL_ITERATOR, NEXT_ITERATOR_ONE); + + assertNoAdvance(dataFetcher, iteratorTwoResults, NEXT_ITERATOR_ONE); + assertAdvanced(dataFetcher, iteratorTwoResults, NEXT_ITERATOR_ONE, NEXT_ITERATOR_TWO); + + assertNoAdvance(dataFetcher, finalResult, NEXT_ITERATOR_TWO); + assertAdvanced(dataFetcher, finalResult, NEXT_ITERATOR_TWO, null); + + verify(kinesisProxy, times(2)).get(eq(INITIAL_ITERATOR), anyInt()); + verify(kinesisProxy, times(2)).get(eq(NEXT_ITERATOR_ONE), anyInt()); + verify(kinesisProxy, times(2)).get(eq(NEXT_ITERATOR_TWO), anyInt()); + + reset(kinesisProxy); + + DataFetcherResult terminal = dataFetcher.getRecords(100); + assertThat(terminal.isShardEnd(), equalTo(true)); + assertThat(terminal.getResult(), notNullValue()); + GetRecordsResult terminalResult = terminal.getResult(); + assertThat(terminalResult.getRecords(), notNullValue()); + assertThat(terminalResult.getRecords(), empty()); + assertThat(terminalResult.getNextShardIterator(), nullValue()); + assertThat(terminal, equalTo(dataFetcher.TERMINAL_RESULT)); + + verify(kinesisProxy, never()).get(anyString(), anyInt()); + } + + @Test + public void testRestartIterator() { + GetRecordsResult getRecordsResult = mock(GetRecordsResult.class); + GetRecordsResult restartGetRecordsResult = new GetRecordsResult(); + Record record = mock(Record.class); + final String initialIterator = "InitialIterator"; + final String nextShardIterator = "NextShardIterator"; + final String restartShardIterator = "RestartIterator"; + final String sequenceNumber = "SequenceNumber"; + final String iteratorType = "AT_SEQUENCE_NUMBER"; + KinesisProxy kinesisProxy = mock(KinesisProxy.class); + KinesisDataFetcher fetcher = new KinesisDataFetcher(kinesisProxy, SHARD_INFO); + + when(kinesisProxy.getIterator(eq(SHARD_ID), eq(InitialPositionInStream.LATEST.toString()))).thenReturn(initialIterator); + when(kinesisProxy.get(eq(initialIterator), eq(10))).thenReturn(getRecordsResult); + when(getRecordsResult.getRecords()).thenReturn(Collections.singletonList(record)); + when(getRecordsResult.getNextShardIterator()).thenReturn(nextShardIterator); + when(record.getSequenceNumber()).thenReturn(sequenceNumber); + + fetcher.initialize(InitialPositionInStream.LATEST.toString(), INITIAL_POSITION_LATEST); + verify(kinesisProxy).getIterator(eq(SHARD_ID), eq(InitialPositionInStream.LATEST.toString())); + Assert.assertEquals(getRecordsResult, fetcher.getRecords(10).accept()); + verify(kinesisProxy).get(eq(initialIterator), eq(10)); + + when(kinesisProxy.getIterator(eq(SHARD_ID), eq(iteratorType), eq(sequenceNumber))).thenReturn(restartShardIterator); + when(kinesisProxy.get(eq(restartShardIterator), eq(10))).thenReturn(restartGetRecordsResult); + + fetcher.restartIterator(); + Assert.assertEquals(restartGetRecordsResult, fetcher.getRecords(10).accept()); + verify(kinesisProxy).getIterator(eq(SHARD_ID), eq(iteratorType), eq(sequenceNumber)); + verify(kinesisProxy).get(eq(restartShardIterator), eq(10)); + } + + @Test (expected = IllegalStateException.class) + public void testRestartIteratorNotInitialized() { + KinesisDataFetcher dataFetcher = new KinesisDataFetcher(kinesisProxy, SHARD_INFO); + dataFetcher.restartIterator(); + } + + private DataFetcherResult assertAdvanced(KinesisDataFetcher dataFetcher, GetRecordsResult expectedResult, + String previousValue, String nextValue) { + DataFetcherResult acceptResult = dataFetcher.getRecords(100); + assertThat(acceptResult.getResult(), equalTo(expectedResult)); + + assertThat(dataFetcher.getNextIterator(), equalTo(previousValue)); + assertThat(dataFetcher.isShardEndReached(), equalTo(false)); + + assertThat(acceptResult.accept(), equalTo(expectedResult)); + assertThat(dataFetcher.getNextIterator(), equalTo(nextValue)); + if (nextValue == null) { + assertThat(dataFetcher.isShardEndReached(), equalTo(true)); + } + + verify(kinesisProxy, times(2)).get(eq(previousValue), anyInt()); + + return acceptResult; + } + + private DataFetcherResult assertNoAdvance(KinesisDataFetcher dataFetcher, GetRecordsResult expectedResult, + String previousValue) { + assertThat(dataFetcher.getNextIterator(), equalTo(previousValue)); + DataFetcherResult noAcceptResult = dataFetcher.getRecords(100); + assertThat(noAcceptResult.getResult(), equalTo(expectedResult)); + + assertThat(dataFetcher.getNextIterator(), equalTo(previousValue)); + + verify(kinesisProxy).get(eq(previousValue), anyInt()); + + return noAcceptResult; + } private void testInitializeAndFetch(String iteratorType, String seqNo, @@ -206,8 +364,9 @@ public class KinesisDataFetcherTest { when(checkpoint.getCheckpoint(SHARD_ID)).thenReturn(new ExtendedSequenceNumber(seqNo)); KinesisDataFetcher fetcher = new KinesisDataFetcher(kinesis, SHARD_INFO); + GetRecordsRetrievalStrategy getRecordsRetrievalStrategy = new SynchronousGetRecordsRetrievalStrategy(fetcher); fetcher.initialize(seqNo, initialPositionInStream); - List actualRecords = fetcher.getRecords(MAX_RECORDS).getRecords(); + List actualRecords = getRecordsRetrievalStrategy.getRecords(MAX_RECORDS).getRecords(); Assert.assertEquals(expectedRecords, actualRecords); } diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ParentsFirstShardPrioritizationUnitTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ParentsFirstShardPrioritizationUnitTest.java index 7ba0753d..42fd82de 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ParentsFirstShardPrioritizationUnitTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ParentsFirstShardPrioritizationUnitTest.java @@ -1,3 +1,17 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; import static org.junit.Assert.assertEquals; diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PrefetchGetRecordsCacheIntegrationTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PrefetchGetRecordsCacheIntegrationTest.java new file mode 100644 index 00000000..e24d5bb0 --- /dev/null +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PrefetchGetRecordsCacheIntegrationTest.java @@ -0,0 +1,222 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazonaws.services.kinesis.clientlibrary.lib.worker; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.runners.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +import com.amazonaws.services.kinesis.clientlibrary.proxies.IKinesisProxy; +import com.amazonaws.services.kinesis.clientlibrary.types.ProcessRecordsInput; +import com.amazonaws.services.kinesis.metrics.impl.NullMetricsFactory; +import com.amazonaws.services.kinesis.model.ExpiredIteratorException; +import com.amazonaws.services.kinesis.model.GetRecordsResult; +import com.amazonaws.services.kinesis.model.Record; + +import lombok.extern.apachecommons.CommonsLog; + +/** + * These are the integration tests for the PrefetchGetRecordsCache class. + */ +@RunWith(MockitoJUnitRunner.class) +@CommonsLog +public class PrefetchGetRecordsCacheIntegrationTest { + private static final int MAX_SIZE = 3; + private static final int MAX_BYTE_SIZE = 5 * 1024 * 1024; + private static final int MAX_RECORDS_COUNT = 30_000; + private static final int MAX_RECORDS_PER_CALL = 10_000; + private static final long IDLE_MILLIS_BETWEEN_CALLS = 500L; + + private PrefetchGetRecordsCache getRecordsCache; + private GetRecordsRetrievalStrategy getRecordsRetrievalStrategy; + private KinesisDataFetcher dataFetcher; + private ExecutorService executorService; + private List records; + private String operation = "ProcessTask"; + + @Mock + private IKinesisProxy proxy; + @Mock + private ShardInfo shardInfo; + + @Before + public void setup() { + records = new ArrayList<>(); + dataFetcher = spy(new KinesisDataFetcherForTest(proxy, shardInfo)); + getRecordsRetrievalStrategy = spy(new SynchronousGetRecordsRetrievalStrategy(dataFetcher)); + executorService = spy(Executors.newFixedThreadPool(1)); + + getRecordsCache = new PrefetchGetRecordsCache(MAX_SIZE, + MAX_BYTE_SIZE, + MAX_RECORDS_COUNT, + MAX_RECORDS_PER_CALL, + getRecordsRetrievalStrategy, + executorService, + IDLE_MILLIS_BETWEEN_CALLS, + new NullMetricsFactory(), + operation, + "test-shard"); + } + + @Test + public void testRollingCache() { + getRecordsCache.start(); + sleep(IDLE_MILLIS_BETWEEN_CALLS); + + ProcessRecordsInput processRecordsInput1 = getRecordsCache.getNextResult(); + + assertTrue(processRecordsInput1.getRecords().isEmpty()); + assertEquals(processRecordsInput1.getMillisBehindLatest(), new Long(1000)); + assertNotNull(processRecordsInput1.getCacheEntryTime()); + + ProcessRecordsInput processRecordsInput2 = getRecordsCache.getNextResult(); + + assertNotEquals(processRecordsInput1, processRecordsInput2); + } + + @Test + public void testFullCache() { + getRecordsCache.start(); + sleep(MAX_SIZE * IDLE_MILLIS_BETWEEN_CALLS); + + assertEquals(getRecordsCache.getRecordsResultQueue.size(), MAX_SIZE); + + ProcessRecordsInput processRecordsInput1 = getRecordsCache.getNextResult(); + ProcessRecordsInput processRecordsInput2 = getRecordsCache.getNextResult(); + + assertNotEquals(processRecordsInput1, processRecordsInput2); + } + + @Test + public void testDifferentShardCaches() { + ExecutorService executorService2 = spy(Executors.newFixedThreadPool(1)); + KinesisDataFetcher kinesisDataFetcher = spy(new KinesisDataFetcherForTest(proxy, shardInfo)); + GetRecordsRetrievalStrategy getRecordsRetrievalStrategy2 = spy(new AsynchronousGetRecordsRetrievalStrategy(kinesisDataFetcher, 5 , 5, "Test-shard")); + GetRecordsCache getRecordsCache2 = new PrefetchGetRecordsCache( + MAX_SIZE, + MAX_BYTE_SIZE, + MAX_RECORDS_COUNT, + MAX_RECORDS_PER_CALL, + getRecordsRetrievalStrategy2, + executorService2, + IDLE_MILLIS_BETWEEN_CALLS, + new NullMetricsFactory(), + operation, + "test-shard-2"); + + getRecordsCache.start(); + sleep(IDLE_MILLIS_BETWEEN_CALLS); + + Record record = mock(Record.class); + ByteBuffer byteBuffer = ByteBuffer.allocate(512 * 1024); + when(record.getData()).thenReturn(byteBuffer); + + records.add(record); + records.add(record); + records.add(record); + records.add(record); + getRecordsCache2.start(); + + sleep(IDLE_MILLIS_BETWEEN_CALLS); + + ProcessRecordsInput p1 = getRecordsCache.getNextResult(); + + ProcessRecordsInput p2 = getRecordsCache2.getNextResult(); + + assertNotEquals(p1, p2); + assertTrue(p1.getRecords().isEmpty()); + assertFalse(p2.getRecords().isEmpty()); + assertEquals(p2.getRecords().size(), records.size()); + + getRecordsCache2.shutdown(); + sleep(100L); + verify(executorService2).shutdownNow(); + verify(getRecordsRetrievalStrategy2).shutdown(); + } + + @Test + public void testExpiredIteratorException() { + when(dataFetcher.getRecords(eq(MAX_RECORDS_PER_CALL))).thenAnswer(new Answer() { + @Override + public DataFetcherResult answer(final InvocationOnMock invocationOnMock) throws Throwable { + throw new ExpiredIteratorException("ExpiredIterator"); + } + }).thenCallRealMethod(); + doNothing().when(dataFetcher).restartIterator(); + + getRecordsCache.start(); + sleep(IDLE_MILLIS_BETWEEN_CALLS); + + ProcessRecordsInput processRecordsInput = getRecordsCache.getNextResult(); + + assertNotNull(processRecordsInput); + assertTrue(processRecordsInput.getRecords().isEmpty()); + verify(dataFetcher).restartIterator(); + } + + @After + public void shutdown() { + getRecordsCache.shutdown(); + sleep(100L); + verify(executorService).shutdownNow(); + verify(getRecordsRetrievalStrategy).shutdown(); + } + + private void sleep(long millis) { + try { + Thread.sleep(millis); + } catch (InterruptedException e) {} + } + + private class KinesisDataFetcherForTest extends KinesisDataFetcher { + public KinesisDataFetcherForTest(final IKinesisProxy kinesisProxy, + final ShardInfo shardInfo) { + super(kinesisProxy, shardInfo); + } + + @Override + public DataFetcherResult getRecords(final int maxRecords) { + GetRecordsResult getRecordsResult = new GetRecordsResult(); + getRecordsResult.setRecords(new ArrayList<>(records)); + getRecordsResult.setMillisBehindLatest(1000L); + + return new AdvancingResult(getRecordsResult); + } + } +} diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PrefetchGetRecordsCacheTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PrefetchGetRecordsCacheTest.java new file mode 100644 index 00000000..2b650866 --- /dev/null +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PrefetchGetRecordsCacheTest.java @@ -0,0 +1,236 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazonaws.services.kinesis.clientlibrary.lib.worker; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.nio.ByteBuffer; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.stream.IntStream; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; + +import com.amazonaws.services.kinesis.clientlibrary.types.ProcessRecordsInput; +import com.amazonaws.services.kinesis.metrics.impl.NullMetricsFactory; +import com.amazonaws.services.kinesis.model.ExpiredIteratorException; +import com.amazonaws.services.kinesis.model.GetRecordsResult; +import com.amazonaws.services.kinesis.model.Record; + +/** + * Test class for the PrefetchGetRecordsCache class. + */ +@RunWith(MockitoJUnitRunner.class) +public class PrefetchGetRecordsCacheTest { + private static final int SIZE_512_KB = 512 * 1024; + private static final int SIZE_1_MB = 2 * SIZE_512_KB; + private static final int MAX_RECORDS_PER_CALL = 10000; + private static final int MAX_SIZE = 5; + private static final int MAX_RECORDS_COUNT = 15000; + private static final long IDLE_MILLIS_BETWEEN_CALLS = 0L; + + @Mock + private GetRecordsRetrievalStrategy getRecordsRetrievalStrategy; + @Mock + private GetRecordsResult getRecordsResult; + @Mock + private Record record; + @Mock + private KinesisDataFetcher dataFetcher; + + private List records; + private ExecutorService executorService; + private LinkedBlockingQueue spyQueue; + private PrefetchGetRecordsCache getRecordsCache; + private String operation = "ProcessTask"; + + @Before + public void setup() { + when(getRecordsRetrievalStrategy.getDataFetcher()).thenReturn(dataFetcher); + + executorService = spy(Executors.newFixedThreadPool(1)); + getRecordsCache = new PrefetchGetRecordsCache( + MAX_SIZE, + 3 * SIZE_1_MB, + MAX_RECORDS_COUNT, + MAX_RECORDS_PER_CALL, + getRecordsRetrievalStrategy, + executorService, + IDLE_MILLIS_BETWEEN_CALLS, + new NullMetricsFactory(), + operation, + "shardId"); + spyQueue = spy(getRecordsCache.getRecordsResultQueue); + records = spy(new ArrayList<>()); + + when(getRecordsRetrievalStrategy.getRecords(eq(MAX_RECORDS_PER_CALL))).thenReturn(getRecordsResult); + when(getRecordsResult.getRecords()).thenReturn(records); + } + + @Test + public void testGetRecords() { + when(records.size()).thenReturn(1000); + when(record.getData()).thenReturn(createByteBufferWithSize(SIZE_512_KB)); + + records.add(record); + records.add(record); + records.add(record); + records.add(record); + records.add(record); + + getRecordsCache.start(); + ProcessRecordsInput result = getRecordsCache.getNextResult(); + + assertEquals(result.getRecords(), records); + + verify(executorService).execute(any()); + verify(getRecordsRetrievalStrategy, atLeast(1)).getRecords(eq(MAX_RECORDS_PER_CALL)); + } + + @Test + public void testFullCacheByteSize() { + when(records.size()).thenReturn(500); + when(record.getData()).thenReturn(createByteBufferWithSize(SIZE_1_MB)); + + records.add(record); + + getRecordsCache.start(); + + // Sleep for a few seconds for the cache to fill up. + sleep(2000); + + verify(getRecordsRetrievalStrategy, times(3)).getRecords(eq(MAX_RECORDS_PER_CALL)); + assertEquals(spyQueue.size(), 3); + } + + @Test + public void testFullCacheRecordsCount() { + int recordsSize = 4500; + when(records.size()).thenReturn(recordsSize); + + getRecordsCache.start(); + + sleep(2000); + + int callRate = (int) Math.ceil((double) MAX_RECORDS_COUNT/recordsSize); + verify(getRecordsRetrievalStrategy, times(callRate)).getRecords(MAX_RECORDS_PER_CALL); + assertEquals(spyQueue.size(), callRate); + assertTrue(callRate < MAX_SIZE); + } + + @Test + public void testFullCacheSize() { + int recordsSize = 200; + when(records.size()).thenReturn(recordsSize); + + getRecordsCache.start(); + + // Sleep for a few seconds for the cache to fill up. + sleep(2000); + + verify(getRecordsRetrievalStrategy, times(MAX_SIZE + 1)).getRecords(eq(MAX_RECORDS_PER_CALL)); + assertEquals(spyQueue.size(), MAX_SIZE); + } + + @Test + public void testMultipleCacheCalls() { + int recordsSize = 20; + when(record.getData()).thenReturn(createByteBufferWithSize(1024)); + + IntStream.range(0, recordsSize).forEach(i -> records.add(record)); + + getRecordsCache.start(); + ProcessRecordsInput processRecordsInput = getRecordsCache.getNextResult(); + + verify(executorService).execute(any()); + assertEquals(processRecordsInput.getRecords(), records); + assertNotNull(processRecordsInput.getCacheEntryTime()); + assertNotNull(processRecordsInput.getCacheExitTime()); + + sleep(2000); + + ProcessRecordsInput processRecordsInput2 = getRecordsCache.getNextResult(); + assertNotEquals(processRecordsInput, processRecordsInput2); + assertEquals(processRecordsInput2.getRecords(), records); + assertNotEquals(processRecordsInput2.getTimeSpentInCache(), Duration.ZERO); + + assertTrue(spyQueue.size() <= MAX_SIZE); + } + + @Test(expected = IllegalStateException.class) + public void testGetNextRecordsWithoutStarting() { + verify(executorService, times(0)).execute(any()); + getRecordsCache.getNextResult(); + } + + @Test(expected = IllegalStateException.class) + public void testCallAfterShutdown() { + when(executorService.isShutdown()).thenReturn(true); + getRecordsCache.getNextResult(); + } + + @Test + public void testExpiredIteratorException() { + getRecordsCache.start(); + + when(getRecordsRetrievalStrategy.getRecords(MAX_RECORDS_PER_CALL)).thenThrow(ExpiredIteratorException.class).thenReturn(getRecordsResult); + doNothing().when(dataFetcher).restartIterator(); + + getRecordsCache.getNextResult(); + + sleep(1000); + + verify(dataFetcher).restartIterator(); + } + + @After + public void shutdown() { + getRecordsCache.shutdown(); + verify(executorService).shutdownNow(); + } + + private void sleep(long millis) { + try { + Thread.sleep(millis); + } catch (InterruptedException e) {} + } + + private ByteBuffer createByteBufferWithSize(int size) { + ByteBuffer byteBuffer = ByteBuffer.allocate(size); + byteBuffer.put(new byte[size]); + return byteBuffer; + } +} diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PreparedCheckpointerTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PreparedCheckpointerTest.java new file mode 100644 index 00000000..bfcd7723 --- /dev/null +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PreparedCheckpointerTest.java @@ -0,0 +1,49 @@ +package com.amazonaws.services.kinesis.clientlibrary.lib.worker; + +import com.amazonaws.services.kinesis.clientlibrary.interfaces.IPreparedCheckpointer; +import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer; +import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +public class PreparedCheckpointerTest { + + /** + * This test verifies the relationship between the constructor and getPendingCheckpoint. + */ + @Test + public void testGetSequenceNumber() { + ExtendedSequenceNumber sn = new ExtendedSequenceNumber("sn"); + IPreparedCheckpointer checkpointer = new PreparedCheckpointer(sn, null); + Assert.assertEquals(sn, checkpointer.getPendingCheckpoint()); + } + + /** + * This test makes sure the PreparedCheckpointer calls the IRecordProcessorCheckpointer properly. + * + * @throws Exception + */ + @Test + public void testCheckpoint() throws Exception { + ExtendedSequenceNumber sn = new ExtendedSequenceNumber("sn"); + IRecordProcessorCheckpointer mockRecordProcessorCheckpointer = Mockito.mock(IRecordProcessorCheckpointer.class); + IPreparedCheckpointer checkpointer = new PreparedCheckpointer(sn, mockRecordProcessorCheckpointer); + checkpointer.checkpoint(); + Mockito.verify(mockRecordProcessorCheckpointer).checkpoint(sn.getSequenceNumber(), sn.getSubSequenceNumber()); + } + + /** + * This test makes sure the PreparedCheckpointer calls the IRecordProcessorCheckpointer properly. + * + * @throws Exception + */ + @Test + public void testDoesNothingPreparedCheckpoint() throws Exception { + ExtendedSequenceNumber sn = new ExtendedSequenceNumber("sn"); + IPreparedCheckpointer checkpointer = new DoesNothingPreparedCheckpointer(sn); + Assert.assertEquals(sn, checkpointer.getPendingCheckpoint()); + // nothing happens here + checkpointer.checkpoint(); + } +} \ No newline at end of file diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ProcessTaskTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ProcessTaskTest.java index e95aef50..94d0918e 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ProcessTaskTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ProcessTaskTest.java @@ -1,16 +1,16 @@ /* - * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; @@ -18,8 +18,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; -import static org.mockito.Matchers.anyInt; -import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.any; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.never; @@ -49,7 +48,6 @@ import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber import com.amazonaws.services.kinesis.clientlibrary.types.Messages.AggregatedRecord; import com.amazonaws.services.kinesis.clientlibrary.types.ProcessRecordsInput; import com.amazonaws.services.kinesis.clientlibrary.types.UserRecord; -import com.amazonaws.services.kinesis.model.GetRecordsResult; import com.amazonaws.services.kinesis.model.ProvisionedThroughputExceededException; import com.amazonaws.services.kinesis.model.Record; import com.google.protobuf.ByteString; @@ -76,6 +74,8 @@ public class ProcessTaskTest { private @Mock RecordProcessorCheckpointer mockCheckpointer; @Mock private ThrottlingReporter throttlingReporter; + @Mock + private GetRecordsCache getRecordsCache; private List processedRecords; private ExtendedSequenceNumber newLargestPermittedCheckpointValue; @@ -93,30 +93,39 @@ public class ProcessTaskTest { INITIAL_POSITION_LATEST); final ShardInfo shardInfo = new ShardInfo(shardId, null, null, null); processTask = new ProcessTask( - shardInfo, config, mockRecordProcessor, mockCheckpointer, mockDataFetcher, taskBackoffTimeMillis, - KinesisClientLibConfiguration.DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST, throttlingReporter); + shardInfo, + config, + mockRecordProcessor, + mockCheckpointer, + mockDataFetcher, + taskBackoffTimeMillis, + KinesisClientLibConfiguration.DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST, + throttlingReporter, + getRecordsCache); } @Test public void testProcessTaskWithProvisionedThroughputExceededException() { // Set data fetcher to throw exception doReturn(false).when(mockDataFetcher).isShardEndReached(); - doThrow(new ProvisionedThroughputExceededException("Test Exception")).when(mockDataFetcher) - .getRecords(maxRecords); + doThrow(new ProvisionedThroughputExceededException("Test Exception")).when(getRecordsCache) + .getNextResult(); TaskResult result = processTask.call(); verify(throttlingReporter).throttled(); verify(throttlingReporter, never()).success(); + verify(getRecordsCache).getNextResult(); assertTrue("Result should contain ProvisionedThroughputExceededException", result.getException() instanceof ProvisionedThroughputExceededException); } @Test public void testProcessTaskWithNonExistentStream() { - // Data fetcher returns a null Result when the stream does not exist - doReturn(null).when(mockDataFetcher).getRecords(maxRecords); + // Data fetcher returns a null Result ` the stream does not exist + doReturn(new ProcessRecordsInput().withRecords(Collections.emptyList()).withMillisBehindLatest((long) 0)).when(getRecordsCache).getNextResult(); TaskResult result = processTask.call(); + verify(getRecordsCache).getNextResult(); assertNull("Task should not throw an exception", result.getException()); } @@ -300,14 +309,13 @@ public class ProcessTaskTest { private void testWithRecords(List records, ExtendedSequenceNumber lastCheckpointValue, ExtendedSequenceNumber largestPermittedCheckpointValue) { - when(mockDataFetcher.getRecords(anyInt())).thenReturn( - new GetRecordsResult().withRecords(records)); + when(getRecordsCache.getNextResult()).thenReturn(new ProcessRecordsInput().withRecords(records).withMillisBehindLatest((long) 1000 * 50)); when(mockCheckpointer.getLastCheckpointValue()).thenReturn(lastCheckpointValue); when(mockCheckpointer.getLargestPermittedCheckpointValue()).thenReturn(largestPermittedCheckpointValue); processTask.call(); verify(throttlingReporter).success(); verify(throttlingReporter, never()).throttled(); - + verify(getRecordsCache).getNextResult(); ArgumentCaptor priCaptor = ArgumentCaptor.forClass(ProcessRecordsInput.class); verify(mockRecordProcessor).processRecords(priCaptor.capture()); processedRecords = priCaptor.getValue().getRecords(); diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/RecordProcessorCheckpointerTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/RecordProcessorCheckpointerTest.java index d5f6b53f..67c36d20 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/RecordProcessorCheckpointerTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/RecordProcessorCheckpointerTest.java @@ -14,42 +14,54 @@ */ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map.Entry; +import com.amazonaws.services.kinesis.metrics.interfaces.IMetricsScope; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; import org.mockito.Mockito; +import org.mockito.runners.MockitoJUnitRunner; -import com.amazonaws.services.kinesis.clientlibrary.exceptions.InvalidStateException; -import com.amazonaws.services.kinesis.clientlibrary.exceptions.KinesisClientLibDependencyException; -import com.amazonaws.services.kinesis.clientlibrary.exceptions.KinesisClientLibException; -import com.amazonaws.services.kinesis.clientlibrary.exceptions.ShutdownException; -import com.amazonaws.services.kinesis.clientlibrary.exceptions.ThrottlingException; import com.amazonaws.services.kinesis.clientlibrary.interfaces.ICheckpoint; +import com.amazonaws.services.kinesis.clientlibrary.interfaces.IPreparedCheckpointer; import com.amazonaws.services.kinesis.clientlibrary.lib.checkpoint.InMemoryCheckpointImpl; import com.amazonaws.services.kinesis.clientlibrary.lib.checkpoint.SentinelCheckpoint; import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber; import com.amazonaws.services.kinesis.clientlibrary.types.UserRecord; +import com.amazonaws.services.kinesis.metrics.impl.MetricsHelper; +import com.amazonaws.services.kinesis.metrics.impl.NullMetricsScope; +import com.amazonaws.services.kinesis.metrics.interfaces.IMetricsFactory; import com.amazonaws.services.kinesis.model.Record; -import static org.junit.Assert.fail; -import static org.mockito.Mockito.mock; -import static org.mockito.Matchers.anyString; - /** * */ +@RunWith(MockitoJUnitRunner.class) public class RecordProcessorCheckpointerTest { private String startingSequenceNumber = "13"; private ExtendedSequenceNumber startingExtendedSequenceNumber = new ExtendedSequenceNumber(startingSequenceNumber); private String testConcurrencyToken = "testToken"; private ICheckpoint checkpoint; + private ShardInfo shardInfo; + private SequenceNumberValidator sequenceNumberValidator; private String shardId = "shardId-123"; + + @Mock + IMetricsFactory metricsFactory; /** * @throws java.lang.Exception @@ -60,6 +72,9 @@ public class RecordProcessorCheckpointerTest { // A real checkpoint will return a checkpoint value after it is initialized. checkpoint.setCheckpoint(shardId, startingExtendedSequenceNumber, testConcurrencyToken); Assert.assertEquals(this.startingExtendedSequenceNumber, checkpoint.getCheckpoint(shardId)); + + shardInfo = new ShardInfo(shardId, testConcurrencyToken, null, ExtendedSequenceNumber.TRIM_HORIZON); + sequenceNumberValidator = new SequenceNumberValidator(null, shardId, false); } /** @@ -75,11 +90,9 @@ public class RecordProcessorCheckpointerTest { */ @Test public final void testCheckpoint() throws Exception { - ShardInfo shardInfo = new ShardInfo(shardId, testConcurrencyToken, null, ExtendedSequenceNumber.TRIM_HORIZON); - // First call to checkpoint RecordProcessorCheckpointer processingCheckpointer = - new RecordProcessorCheckpointer(shardInfo, checkpoint, null); + new RecordProcessorCheckpointer(shardInfo, checkpoint, null, metricsFactory); processingCheckpointer.setLargestPermittedCheckpointValue(startingExtendedSequenceNumber); processingCheckpointer.checkpoint(); Assert.assertEquals(startingExtendedSequenceNumber, checkpoint.getCheckpoint(shardId)); @@ -98,11 +111,8 @@ public class RecordProcessorCheckpointerTest { */ @Test public final void testCheckpointRecord() throws Exception { - ShardInfo shardInfo = new ShardInfo(shardId, testConcurrencyToken, null, ExtendedSequenceNumber.TRIM_HORIZON); - SequenceNumberValidator sequenceNumberValidator = - new SequenceNumberValidator(null, shardId, false); RecordProcessorCheckpointer processingCheckpointer = - new RecordProcessorCheckpointer(shardInfo, checkpoint, sequenceNumberValidator); + new RecordProcessorCheckpointer(shardInfo, checkpoint, sequenceNumberValidator, metricsFactory); processingCheckpointer.setInitialCheckpointValue(startingExtendedSequenceNumber); ExtendedSequenceNumber extendedSequenceNumber = new ExtendedSequenceNumber("5025"); Record record = new Record().withSequenceNumber("5025"); @@ -113,15 +123,12 @@ public class RecordProcessorCheckpointerTest { /** * Test method for - * {@link com.amazonaws.services.kinesis.clientlibrary.lib.worker.RecordProcessorCheckpointer#checkpoint(UserRecord record)}. + * {@link com.amazonaws.services.kinesis.clientlibrary.lib.worker.RecordProcessorCheckpointer#checkpoint(Record record)}. */ @Test public final void testCheckpointSubRecord() throws Exception { - ShardInfo shardInfo = new ShardInfo(shardId, testConcurrencyToken, null, ExtendedSequenceNumber.TRIM_HORIZON); - SequenceNumberValidator sequenceNumberValidator = - new SequenceNumberValidator(null, shardId, false); RecordProcessorCheckpointer processingCheckpointer = - new RecordProcessorCheckpointer(shardInfo, checkpoint, sequenceNumberValidator); + new RecordProcessorCheckpointer(shardInfo, checkpoint, sequenceNumberValidator, metricsFactory); processingCheckpointer.setInitialCheckpointValue(startingExtendedSequenceNumber); ExtendedSequenceNumber extendedSequenceNumber = new ExtendedSequenceNumber("5030"); Record record = new Record().withSequenceNumber("5030"); @@ -137,11 +144,8 @@ public class RecordProcessorCheckpointerTest { */ @Test public final void testCheckpointSequenceNumber() throws Exception { - ShardInfo shardInfo = new ShardInfo(shardId, testConcurrencyToken, null, ExtendedSequenceNumber.TRIM_HORIZON); - SequenceNumberValidator sequenceNumberValidator = - new SequenceNumberValidator(null, shardId, false); RecordProcessorCheckpointer processingCheckpointer = - new RecordProcessorCheckpointer(shardInfo, checkpoint, sequenceNumberValidator); + new RecordProcessorCheckpointer(shardInfo, checkpoint, sequenceNumberValidator, metricsFactory); processingCheckpointer.setInitialCheckpointValue(startingExtendedSequenceNumber); ExtendedSequenceNumber extendedSequenceNumber = new ExtendedSequenceNumber("5035"); processingCheckpointer.setLargestPermittedCheckpointValue(extendedSequenceNumber); @@ -155,11 +159,8 @@ public class RecordProcessorCheckpointerTest { */ @Test public final void testCheckpointExtendedSequenceNumber() throws Exception { - ShardInfo shardInfo = new ShardInfo(shardId, testConcurrencyToken, null, ExtendedSequenceNumber.TRIM_HORIZON); - SequenceNumberValidator sequenceNumberValidator = - new SequenceNumberValidator(null, shardId, false); RecordProcessorCheckpointer processingCheckpointer = - new RecordProcessorCheckpointer(shardInfo, checkpoint, sequenceNumberValidator); + new RecordProcessorCheckpointer(shardInfo, checkpoint, sequenceNumberValidator, metricsFactory); processingCheckpointer.setInitialCheckpointValue(startingExtendedSequenceNumber); ExtendedSequenceNumber extendedSequenceNumber = new ExtendedSequenceNumber("5040"); processingCheckpointer.setLargestPermittedCheckpointValue(extendedSequenceNumber); @@ -167,15 +168,250 @@ public class RecordProcessorCheckpointerTest { Assert.assertEquals(extendedSequenceNumber, checkpoint.getCheckpoint(shardId)); } + /** + * Test method for {@link RecordProcessorCheckpointer#checkpoint(String SHARD_END)}. + */ + @Test + public final void testCheckpointAtShardEnd() throws Exception { + RecordProcessorCheckpointer processingCheckpointer = + new RecordProcessorCheckpointer(shardInfo, checkpoint, sequenceNumberValidator, metricsFactory); + processingCheckpointer.setInitialCheckpointValue(startingExtendedSequenceNumber); + ExtendedSequenceNumber extendedSequenceNumber = ExtendedSequenceNumber.SHARD_END; + processingCheckpointer.setLargestPermittedCheckpointValue(extendedSequenceNumber); + processingCheckpointer.checkpoint(ExtendedSequenceNumber.SHARD_END.getSequenceNumber()); + Assert.assertEquals(extendedSequenceNumber, checkpoint.getCheckpoint(shardId)); + } + + + /** + * Test method for + * {@link com.amazonaws.services.kinesis.clientlibrary.lib.worker.RecordProcessorCheckpointer#prepareCheckpoint()}. + */ + @Test + public final void testPrepareCheckpoint() throws Exception { + // First call to checkpoint + RecordProcessorCheckpointer processingCheckpointer = + new RecordProcessorCheckpointer(shardInfo, checkpoint, sequenceNumberValidator, metricsFactory); + processingCheckpointer.setInitialCheckpointValue(startingExtendedSequenceNumber); + + ExtendedSequenceNumber sequenceNumber1 = new ExtendedSequenceNumber("5001"); + processingCheckpointer.setLargestPermittedCheckpointValue(sequenceNumber1); + IPreparedCheckpointer preparedCheckpoint = processingCheckpointer.prepareCheckpoint(); + Assert.assertEquals(sequenceNumber1, preparedCheckpoint.getPendingCheckpoint()); + Assert.assertEquals(sequenceNumber1, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + + // Advance checkpoint + ExtendedSequenceNumber sequenceNumber2 = new ExtendedSequenceNumber("5019"); + + processingCheckpointer.setLargestPermittedCheckpointValue(sequenceNumber2); + preparedCheckpoint = processingCheckpointer.prepareCheckpoint(); + Assert.assertEquals(sequenceNumber2, preparedCheckpoint.getPendingCheckpoint()); + Assert.assertEquals(sequenceNumber2, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + + // Checkpoint using preparedCheckpoint + preparedCheckpoint.checkpoint(); + Assert.assertEquals(sequenceNumber2, checkpoint.getCheckpoint(shardId)); + Assert.assertEquals(sequenceNumber2, checkpoint.getCheckpointObject(shardId).getCheckpoint()); + Assert.assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + } + + /** + * Test method for + * {@link com.amazonaws.services.kinesis.clientlibrary.lib.worker.RecordProcessorCheckpointer#prepareCheckpoint(Record record)}. + */ + @Test + public final void testPrepareCheckpointRecord() throws Exception { + RecordProcessorCheckpointer processingCheckpointer = + new RecordProcessorCheckpointer(shardInfo, checkpoint, sequenceNumberValidator, metricsFactory); + processingCheckpointer.setInitialCheckpointValue(startingExtendedSequenceNumber); + ExtendedSequenceNumber extendedSequenceNumber = new ExtendedSequenceNumber("5025"); + Record record = new Record().withSequenceNumber("5025"); + processingCheckpointer.setLargestPermittedCheckpointValue(extendedSequenceNumber); + IPreparedCheckpointer preparedCheckpoint = processingCheckpointer.prepareCheckpoint(record); + Assert.assertEquals(startingExtendedSequenceNumber, checkpoint.getCheckpoint(shardId)); + Assert.assertEquals(startingExtendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); + Assert.assertEquals(extendedSequenceNumber, preparedCheckpoint.getPendingCheckpoint()); + Assert.assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + + // Checkpoint using preparedCheckpoint + preparedCheckpoint.checkpoint(); + Assert.assertEquals(extendedSequenceNumber, checkpoint.getCheckpoint(shardId)); + Assert.assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); + Assert.assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + } + + /** + * Test method for + * {@link com.amazonaws.services.kinesis.clientlibrary.lib.worker.RecordProcessorCheckpointer#prepareCheckpoint(Record record)}. + */ + @Test + public final void testPrepareCheckpointSubRecord() throws Exception { + RecordProcessorCheckpointer processingCheckpointer = + new RecordProcessorCheckpointer(shardInfo, checkpoint, sequenceNumberValidator, metricsFactory); + processingCheckpointer.setInitialCheckpointValue(startingExtendedSequenceNumber); + ExtendedSequenceNumber extendedSequenceNumber = new ExtendedSequenceNumber("5030"); + Record record = new Record().withSequenceNumber("5030"); + UserRecord subRecord = new UserRecord(record); + processingCheckpointer.setLargestPermittedCheckpointValue(extendedSequenceNumber); + IPreparedCheckpointer preparedCheckpoint = processingCheckpointer.prepareCheckpoint(subRecord); + Assert.assertEquals(startingExtendedSequenceNumber, checkpoint.getCheckpoint(shardId)); + Assert.assertEquals(startingExtendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); + Assert.assertEquals(extendedSequenceNumber, preparedCheckpoint.getPendingCheckpoint()); + Assert.assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + + // Checkpoint using preparedCheckpoint + preparedCheckpoint.checkpoint(); + Assert.assertEquals(extendedSequenceNumber, checkpoint.getCheckpoint(shardId)); + Assert.assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); + Assert.assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + } + + /** + * Test method for + * {@link com.amazonaws.services.kinesis.clientlibrary.lib.worker.RecordProcessorCheckpointer#checkpoint(String sequenceNumber)}. + */ + @Test + public final void testPrepareCheckpointSequenceNumber() throws Exception { + RecordProcessorCheckpointer processingCheckpointer = + new RecordProcessorCheckpointer(shardInfo, checkpoint, sequenceNumberValidator, metricsFactory); + processingCheckpointer.setInitialCheckpointValue(startingExtendedSequenceNumber); + ExtendedSequenceNumber extendedSequenceNumber = new ExtendedSequenceNumber("5035"); + processingCheckpointer.setLargestPermittedCheckpointValue(extendedSequenceNumber); + IPreparedCheckpointer preparedCheckpoint = processingCheckpointer.prepareCheckpoint("5035"); + Assert.assertEquals(startingExtendedSequenceNumber, checkpoint.getCheckpoint(shardId)); + Assert.assertEquals(startingExtendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); + Assert.assertEquals(extendedSequenceNumber, preparedCheckpoint.getPendingCheckpoint()); + Assert.assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + + // Checkpoint using preparedCheckpoint + preparedCheckpoint.checkpoint(); + Assert.assertEquals(extendedSequenceNumber, checkpoint.getCheckpoint(shardId)); + Assert.assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); + Assert.assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + } + + /** + * Test method for + * {@link com.amazonaws.services.kinesis.clientlibrary.lib.worker.RecordProcessorCheckpointer#checkpoint(String sequenceNumber, long subSequenceNumber)}. + */ + @Test + public final void testPrepareCheckpointExtendedSequenceNumber() throws Exception { + RecordProcessorCheckpointer processingCheckpointer = + new RecordProcessorCheckpointer(shardInfo, checkpoint, sequenceNumberValidator, metricsFactory); + processingCheckpointer.setInitialCheckpointValue(startingExtendedSequenceNumber); + ExtendedSequenceNumber extendedSequenceNumber = new ExtendedSequenceNumber("5040"); + processingCheckpointer.setLargestPermittedCheckpointValue(extendedSequenceNumber); + IPreparedCheckpointer preparedCheckpoint = processingCheckpointer.prepareCheckpoint("5040", 0); + Assert.assertEquals(startingExtendedSequenceNumber, checkpoint.getCheckpoint(shardId)); + Assert.assertEquals(startingExtendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); + Assert.assertEquals(extendedSequenceNumber, preparedCheckpoint.getPendingCheckpoint()); + Assert.assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + + // Checkpoint using preparedCheckpoint + preparedCheckpoint.checkpoint(); + Assert.assertEquals(extendedSequenceNumber, checkpoint.getCheckpoint(shardId)); + Assert.assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); + Assert.assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + } + + /** + * Test method for {@link RecordProcessorCheckpointer#checkpoint(String SHARD_END)}. + */ + @Test + public final void testPrepareCheckpointAtShardEnd() throws Exception { + RecordProcessorCheckpointer processingCheckpointer = + new RecordProcessorCheckpointer(shardInfo, checkpoint, sequenceNumberValidator, metricsFactory); + processingCheckpointer.setInitialCheckpointValue(startingExtendedSequenceNumber); + ExtendedSequenceNumber extendedSequenceNumber = ExtendedSequenceNumber.SHARD_END; + processingCheckpointer.setLargestPermittedCheckpointValue(extendedSequenceNumber); + IPreparedCheckpointer preparedCheckpoint = processingCheckpointer.prepareCheckpoint(ExtendedSequenceNumber.SHARD_END.getSequenceNumber()); + Assert.assertEquals(startingExtendedSequenceNumber, checkpoint.getCheckpoint(shardId)); + Assert.assertEquals(startingExtendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); + Assert.assertEquals(extendedSequenceNumber, preparedCheckpoint.getPendingCheckpoint()); + Assert.assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + + // Checkpoint using preparedCheckpoint + preparedCheckpoint.checkpoint(); + Assert.assertEquals(extendedSequenceNumber, checkpoint.getCheckpoint(shardId)); + Assert.assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); + Assert.assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + } + + + /** + * Test that having multiple outstanding prepared checkpointers works if they are redeemed in the right order. + */ + @Test + public final void testMultipleOutstandingCheckpointersHappyCase() throws Exception { + RecordProcessorCheckpointer processingCheckpointer = + new RecordProcessorCheckpointer(shardInfo, checkpoint, sequenceNumberValidator, metricsFactory); + processingCheckpointer.setInitialCheckpointValue(startingExtendedSequenceNumber); + processingCheckpointer.setLargestPermittedCheckpointValue(new ExtendedSequenceNumber("6040")); + + ExtendedSequenceNumber sn1 = new ExtendedSequenceNumber("6010"); + IPreparedCheckpointer firstPreparedCheckpoint = processingCheckpointer.prepareCheckpoint("6010", 0); + Assert.assertEquals(sn1, firstPreparedCheckpoint.getPendingCheckpoint()); + Assert.assertEquals(sn1, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + + ExtendedSequenceNumber sn2 = new ExtendedSequenceNumber("6020"); + IPreparedCheckpointer secondPreparedCheckpoint = processingCheckpointer.prepareCheckpoint("6020", 0); + Assert.assertEquals(sn2, secondPreparedCheckpoint.getPendingCheckpoint()); + Assert.assertEquals(sn2, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + + // checkpoint in order + firstPreparedCheckpoint.checkpoint(); + Assert.assertEquals(sn1, checkpoint.getCheckpoint(shardId)); + Assert.assertEquals(sn1, checkpoint.getCheckpointObject(shardId).getCheckpoint()); + Assert.assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + + secondPreparedCheckpoint.checkpoint(); + Assert.assertEquals(sn2, checkpoint.getCheckpoint(shardId)); + Assert.assertEquals(sn2, checkpoint.getCheckpointObject(shardId).getCheckpoint()); + Assert.assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + } + + /** + * Test that having multiple outstanding prepared checkpointers works if they are redeemed in the right order. + */ + @Test + public final void testMultipleOutstandingCheckpointersOutOfOrder() throws Exception { + RecordProcessorCheckpointer processingCheckpointer = + new RecordProcessorCheckpointer(shardInfo, checkpoint, sequenceNumberValidator, metricsFactory); + processingCheckpointer.setInitialCheckpointValue(startingExtendedSequenceNumber); + processingCheckpointer.setLargestPermittedCheckpointValue(new ExtendedSequenceNumber("7040")); + + ExtendedSequenceNumber sn1 = new ExtendedSequenceNumber("7010"); + IPreparedCheckpointer firstPreparedCheckpoint = processingCheckpointer.prepareCheckpoint("7010", 0); + Assert.assertEquals(sn1, firstPreparedCheckpoint.getPendingCheckpoint()); + Assert.assertEquals(sn1, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + + ExtendedSequenceNumber sn2 = new ExtendedSequenceNumber("7020"); + IPreparedCheckpointer secondPreparedCheckpoint = processingCheckpointer.prepareCheckpoint("7020", 0); + Assert.assertEquals(sn2, secondPreparedCheckpoint.getPendingCheckpoint()); + Assert.assertEquals(sn2, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + + // checkpoint out of order + secondPreparedCheckpoint.checkpoint(); + Assert.assertEquals(sn2, checkpoint.getCheckpoint(shardId)); + Assert.assertEquals(sn2, checkpoint.getCheckpointObject(shardId).getCheckpoint()); + Assert.assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + + try { + firstPreparedCheckpoint.checkpoint(); + Assert.fail("checkpoint() should have failed because the sequence number was too low"); + } catch (IllegalArgumentException e) { + } catch (Exception e) { + Assert.fail("checkpoint() should have thrown an IllegalArgumentException but instead threw " + e); + } + } + /** * Test method for update() * */ @Test public final void testUpdate() throws Exception { - ShardInfo shardInfo = new ShardInfo(shardId, testConcurrencyToken, null, ExtendedSequenceNumber.TRIM_HORIZON); - - RecordProcessorCheckpointer checkpointer = new RecordProcessorCheckpointer(shardInfo, checkpoint, null); + RecordProcessorCheckpointer checkpointer = new RecordProcessorCheckpointer(shardInfo, checkpoint, null, metricsFactory); ExtendedSequenceNumber sequenceNumber = new ExtendedSequenceNumber("10"); checkpointer.setLargestPermittedCheckpointValue(sequenceNumber); @@ -193,12 +429,10 @@ public class RecordProcessorCheckpointerTest { */ @Test public final void testClientSpecifiedCheckpoint() throws Exception { - ShardInfo shardInfo = new ShardInfo(shardId, testConcurrencyToken, null, ExtendedSequenceNumber.TRIM_HORIZON); - SequenceNumberValidator validator = mock(SequenceNumberValidator.class); Mockito.doNothing().when(validator).validateSequenceNumber(anyString()); RecordProcessorCheckpointer processingCheckpointer = - new RecordProcessorCheckpointer(shardInfo, checkpoint, validator); + new RecordProcessorCheckpointer(shardInfo, checkpoint, validator, metricsFactory); // Several checkpoints we're gonna hit ExtendedSequenceNumber tooSmall = new ExtendedSequenceNumber("2"); @@ -275,10 +509,129 @@ public class RecordProcessorCheckpointerTest { processingCheckpointer.getLastCheckpointValue()); } + /* + * This test is a mixed test of checking some basic functionality of two phase checkpointing at a sequence number + * and making sure certain bounds checks and validations are being performed inside the checkpointer to prevent + * clients from checkpointing out of order/too big/non-numeric values that aren't valid strings for them to be + * checkpointing + */ + @Test + public final void testClientSpecifiedTwoPhaseCheckpoint() throws Exception { + SequenceNumberValidator validator = mock(SequenceNumberValidator.class); + Mockito.doNothing().when(validator).validateSequenceNumber(anyString()); + RecordProcessorCheckpointer processingCheckpointer = + new RecordProcessorCheckpointer(shardInfo, checkpoint, validator, metricsFactory); + + // Several checkpoints we're gonna hit + ExtendedSequenceNumber tooSmall = new ExtendedSequenceNumber("2"); + ExtendedSequenceNumber firstSequenceNumber = checkpoint.getCheckpoint(shardId); // 13 + ExtendedSequenceNumber secondSequenceNumber = new ExtendedSequenceNumber("127"); + ExtendedSequenceNumber thirdSequenceNumber = new ExtendedSequenceNumber("5019"); + ExtendedSequenceNumber lastSequenceNumberOfShard = new ExtendedSequenceNumber("6789"); + ExtendedSequenceNumber tooBigSequenceNumber = new ExtendedSequenceNumber("9000"); + + processingCheckpointer.setInitialCheckpointValue(firstSequenceNumber); + processingCheckpointer.setLargestPermittedCheckpointValue(thirdSequenceNumber); + + // confirm that we cannot move backward + try { + processingCheckpointer.prepareCheckpoint(tooSmall.getSequenceNumber(), tooSmall.getSubSequenceNumber()); + Assert.fail("You shouldn't be able to prepare a checkpoint earlier than the initial checkpoint."); + } catch (IllegalArgumentException e) { + // yay! + } + + try { + processingCheckpointer.checkpoint(tooSmall.getSequenceNumber(), tooSmall.getSubSequenceNumber()); + Assert.fail("You shouldn't be able to checkpoint earlier than the initial checkpoint."); + } catch (IllegalArgumentException e) { + // yay! + } + + // advance to first + processingCheckpointer.checkpoint(firstSequenceNumber.getSequenceNumber(), firstSequenceNumber.getSubSequenceNumber()); + Assert.assertEquals(firstSequenceNumber, checkpoint.getCheckpoint(shardId)); + + // prepare checkpoint at initial checkpoint value + IPreparedCheckpointer doesNothingPreparedCheckpoint = + processingCheckpointer.prepareCheckpoint(firstSequenceNumber.getSequenceNumber(), firstSequenceNumber.getSubSequenceNumber()); + Assert.assertTrue(doesNothingPreparedCheckpoint instanceof DoesNothingPreparedCheckpointer); + Assert.assertEquals(firstSequenceNumber, doesNothingPreparedCheckpoint.getPendingCheckpoint()); + Assert.assertEquals(firstSequenceNumber, checkpoint.getCheckpoint(shardId)); + Assert.assertEquals(firstSequenceNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); + Assert.assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + + // nothing happens after checkpointing a doesNothingPreparedCheckpoint + doesNothingPreparedCheckpoint.checkpoint(); + Assert.assertEquals(firstSequenceNumber, checkpoint.getCheckpoint(shardId)); + Assert.assertEquals(firstSequenceNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); + Assert.assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + + // advance to second + processingCheckpointer.prepareCheckpoint(secondSequenceNumber.getSequenceNumber(), secondSequenceNumber.getSubSequenceNumber()); + Assert.assertEquals(secondSequenceNumber, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + processingCheckpointer.checkpoint(secondSequenceNumber.getSequenceNumber(), secondSequenceNumber.getSubSequenceNumber()); + Assert.assertEquals(secondSequenceNumber, checkpoint.getCheckpoint(shardId)); + Assert.assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + + ExtendedSequenceNumber[] valuesWeShouldNotBeAbleToCheckpointAt = + { tooSmall, // Shouldn't be able to move before the first value we ever checkpointed + firstSequenceNumber, // Shouldn't even be able to move back to a once used sequence number + tooBigSequenceNumber, // Can't exceed the max sequence number in the checkpointer + lastSequenceNumberOfShard, // Just another big value that we will use later + null, // Not a valid sequence number + new ExtendedSequenceNumber("bogus-checkpoint-value"), // Can't checkpoint at non-numeric string + ExtendedSequenceNumber.SHARD_END, // Can't go to the end unless it is set as the max + ExtendedSequenceNumber.TRIM_HORIZON, // Can't go back to an initial sentinel value + ExtendedSequenceNumber.LATEST // Can't go back to an initial sentinel value + }; + for (ExtendedSequenceNumber badCheckpointValue : valuesWeShouldNotBeAbleToCheckpointAt) { + try { + processingCheckpointer.prepareCheckpoint(badCheckpointValue.getSequenceNumber(), badCheckpointValue.getSubSequenceNumber()); + fail("checkpointing at bad or out of order sequence didn't throw exception"); + } catch (IllegalArgumentException e) { + + } catch (NullPointerException e) { + + } + Assert.assertEquals("Checkpoint value should not have changed", + secondSequenceNumber, + checkpoint.getCheckpoint(shardId)); + Assert.assertEquals("Last checkpoint value should not have changed", + secondSequenceNumber, + processingCheckpointer.getLastCheckpointValue()); + Assert.assertEquals("Largest sequence number should not have changed", + thirdSequenceNumber, + processingCheckpointer.getLargestPermittedCheckpointValue()); + Assert.assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + + } + + // advance to third number + processingCheckpointer.prepareCheckpoint(thirdSequenceNumber.getSequenceNumber(), thirdSequenceNumber.getSubSequenceNumber()); + Assert.assertEquals(thirdSequenceNumber, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + processingCheckpointer.checkpoint(thirdSequenceNumber.getSequenceNumber(), thirdSequenceNumber.getSubSequenceNumber()); + Assert.assertEquals(thirdSequenceNumber, checkpoint.getCheckpoint(shardId)); + + // Testing a feature that prevents checkpointing at SHARD_END twice + processingCheckpointer.setLargestPermittedCheckpointValue(lastSequenceNumberOfShard); + processingCheckpointer.setSequenceNumberAtShardEnd(processingCheckpointer.getLargestPermittedCheckpointValue()); + processingCheckpointer.setLargestPermittedCheckpointValue(ExtendedSequenceNumber.SHARD_END); + processingCheckpointer.prepareCheckpoint(lastSequenceNumberOfShard.getSequenceNumber(), lastSequenceNumberOfShard.getSubSequenceNumber()); + Assert.assertEquals("Preparing a checkpoing at the sequence number at the end of a shard should be the same as " + + "preparing a checkpoint at SHARD_END", + ExtendedSequenceNumber.SHARD_END, + checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + } + private enum CheckpointAction { NONE, NO_SEQUENCE_NUMBER, WITH_SEQUENCE_NUMBER; } + private enum CheckpointerType { + CHECKPOINTER, PREPARED_CHECKPOINTER, PREPARE_THEN_CHECKPOINTER; + } + /** * Tests a bunch of mixed calls between checkpoint() and checkpoint(sequenceNumber) using a helper function. * @@ -290,16 +643,59 @@ public class RecordProcessorCheckpointerTest { @SuppressWarnings("serial") @Test public final void testMixedCheckpointCalls() throws Exception { - ShardInfo shardInfo = new ShardInfo(shardId, testConcurrencyToken, null, ExtendedSequenceNumber.TRIM_HORIZON); - SequenceNumberValidator validator = mock(SequenceNumberValidator.class); Mockito.doNothing().when(validator).validateSequenceNumber(anyString()); - RecordProcessorCheckpointer processingCheckpointer = - new RecordProcessorCheckpointer(shardInfo, checkpoint, validator); + for (LinkedHashMap testPlan : getMixedCallsTestPlan()) { + RecordProcessorCheckpointer processingCheckpointer = + new RecordProcessorCheckpointer(shardInfo, checkpoint, validator, metricsFactory); + testMixedCheckpointCalls(processingCheckpointer, testPlan, CheckpointerType.CHECKPOINTER); + } + } - List> testPlans = - new ArrayList>(); + /** + * similar to + * {@link RecordProcessorCheckpointerTest#testMixedCheckpointCalls()} , + * but executes in two phase commit mode, where we prepare a checkpoint and then commit the prepared checkpoint + * + * @throws Exception + */ + @SuppressWarnings("serial") + @Test + public final void testMixedTwoPhaseCheckpointCalls() throws Exception { + SequenceNumberValidator validator = mock(SequenceNumberValidator.class); + Mockito.doNothing().when(validator).validateSequenceNumber(anyString()); + + for (LinkedHashMap testPlan : getMixedCallsTestPlan()) { + RecordProcessorCheckpointer processingCheckpointer = + new RecordProcessorCheckpointer(shardInfo, checkpoint, validator, metricsFactory); + testMixedCheckpointCalls(processingCheckpointer, testPlan, CheckpointerType.PREPARED_CHECKPOINTER); + } + } + + /** + * similar to + * {@link RecordProcessorCheckpointerTest#testMixedCheckpointCalls()} , + * but executes in two phase commit mode, where we prepare a checkpoint, but we checkpoint using the + * RecordProcessorCheckpointer instead of the returned IPreparedCheckpointer + * + * @throws Exception + */ + @SuppressWarnings("serial") + @Test + public final void testMixedTwoPhaseCheckpointCalls2() throws Exception { + SequenceNumberValidator validator = mock(SequenceNumberValidator.class); + Mockito.doNothing().when(validator).validateSequenceNumber(anyString()); + + for (LinkedHashMap testPlan : getMixedCallsTestPlan()) { + RecordProcessorCheckpointer processingCheckpointer = + new RecordProcessorCheckpointer(shardInfo, checkpoint, validator, metricsFactory); + testMixedCheckpointCalls(processingCheckpointer, testPlan, CheckpointerType.PREPARE_THEN_CHECKPOINTER); + } + } + + private List> getMixedCallsTestPlan() { + List> testPlans = new ArrayList>(); /* * Simulate a scenario where the checkpointer is created at "latest". @@ -356,11 +752,7 @@ public class RecordProcessorCheckpointerTest { } }); - for (LinkedHashMap testPlan : testPlans) { - processingCheckpointer = - new RecordProcessorCheckpointer(shardInfo, checkpoint, validator); - testMixedCheckpointCalls(processingCheckpointer, testPlan); - } + return testPlans; } /** @@ -376,9 +768,11 @@ public class RecordProcessorCheckpointerTest { * @throws Exception */ private void testMixedCheckpointCalls(RecordProcessorCheckpointer processingCheckpointer, - LinkedHashMap checkpointValueAndAction) throws Exception { + LinkedHashMap checkpointValueAndAction, + CheckpointerType checkpointerType) throws Exception { for (Entry entry : checkpointValueAndAction.entrySet()) { + IPreparedCheckpointer preparedCheckpoint = null; ExtendedSequenceNumber lastCheckpointValue = processingCheckpointer.getLastCheckpointValue(); if (SentinelCheckpoint.SHARD_END.toString().equals(entry.getKey())) { @@ -400,10 +794,34 @@ public class RecordProcessorCheckpointerTest { processingCheckpointer.getLastCheckpointValue()); continue; case NO_SEQUENCE_NUMBER: - processingCheckpointer.checkpoint(); + switch (checkpointerType) { + case CHECKPOINTER: + processingCheckpointer.checkpoint(); + break; + case PREPARED_CHECKPOINTER: + preparedCheckpoint = processingCheckpointer.prepareCheckpoint(); + preparedCheckpoint.checkpoint(); + case PREPARE_THEN_CHECKPOINTER: + preparedCheckpoint = processingCheckpointer.prepareCheckpoint(); + processingCheckpointer.checkpoint( + preparedCheckpoint.getPendingCheckpoint().getSequenceNumber(), + preparedCheckpoint.getPendingCheckpoint().getSubSequenceNumber()); + } break; case WITH_SEQUENCE_NUMBER: - processingCheckpointer.checkpoint(entry.getKey()); + switch (checkpointerType) { + case CHECKPOINTER: + processingCheckpointer.checkpoint(entry.getKey()); + break; + case PREPARED_CHECKPOINTER: + preparedCheckpoint = processingCheckpointer.prepareCheckpoint(entry.getKey()); + preparedCheckpoint.checkpoint(); + case PREPARE_THEN_CHECKPOINTER: + preparedCheckpoint = processingCheckpointer.prepareCheckpoint(entry.getKey()); + processingCheckpointer.checkpoint( + preparedCheckpoint.getPendingCheckpoint().getSequenceNumber(), + preparedCheckpoint.getPendingCheckpoint().getSubSequenceNumber()); + } break; } // We must have checkpointed to get here, so let's make sure our last checkpoint value is up to date @@ -413,6 +831,54 @@ public class RecordProcessorCheckpointerTest { Assert.assertEquals("Expected the largest checkpoint value to remain the same since the last set", new ExtendedSequenceNumber(entry.getKey()), processingCheckpointer.getLargestPermittedCheckpointValue()); + + Assert.assertEquals(new ExtendedSequenceNumber(entry.getKey()), checkpoint.getCheckpoint(shardId)); + Assert.assertEquals(new ExtendedSequenceNumber(entry.getKey()), + checkpoint.getCheckpointObject(shardId).getCheckpoint()); + Assert.assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + } + } + + @Test + public final void testUnsetMetricsScopeDuringCheckpointing() throws Exception { + // First call to checkpoint + RecordProcessorCheckpointer processingCheckpointer = + new RecordProcessorCheckpointer(shardInfo, checkpoint, null, metricsFactory); + IMetricsScope scope = null; + if (MetricsHelper.isMetricsScopePresent()) { + scope = MetricsHelper.getMetricsScope(); + MetricsHelper.unsetMetricsScope(); + } + ExtendedSequenceNumber sequenceNumber = new ExtendedSequenceNumber("5019"); + processingCheckpointer.setLargestPermittedCheckpointValue(sequenceNumber); + processingCheckpointer.checkpoint(); + Assert.assertEquals(sequenceNumber, checkpoint.getCheckpoint(shardId)); + verify(metricsFactory).createMetrics(); + Assert.assertFalse(MetricsHelper.isMetricsScopePresent()); + if (scope != null) { + MetricsHelper.setMetricsScope(scope); + } + } + + @Test + public final void testSetMetricsScopeDuringCheckpointing() throws Exception { + // First call to checkpoint + RecordProcessorCheckpointer processingCheckpointer = + new RecordProcessorCheckpointer(shardInfo, checkpoint, null, metricsFactory); + boolean shouldUnset = false; + if (!MetricsHelper.isMetricsScopePresent()) { + shouldUnset = true; + MetricsHelper.setMetricsScope(new NullMetricsScope()); + } + ExtendedSequenceNumber sequenceNumber = new ExtendedSequenceNumber("5019"); + processingCheckpointer.setLargestPermittedCheckpointValue(sequenceNumber); + processingCheckpointer.checkpoint(); + Assert.assertEquals(sequenceNumber, checkpoint.getCheckpoint(shardId)); + verify(metricsFactory, never()).createMetrics(); + Assert.assertTrue(MetricsHelper.isMetricsScopePresent()); + assertEquals(NullMetricsScope.class, MetricsHelper.getMetricsScope().getClass()); + if (shouldUnset) { + MetricsHelper.unsetMetricsScope(); } } } diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/RecordsFetcherFactoryTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/RecordsFetcherFactoryTest.java new file mode 100644 index 00000000..d686c914 --- /dev/null +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/RecordsFetcherFactoryTest.java @@ -0,0 +1,43 @@ +package com.amazonaws.services.kinesis.clientlibrary.lib.worker; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.MatcherAssert.assertThat; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import com.amazonaws.services.kinesis.metrics.interfaces.IMetricsFactory; + +public class RecordsFetcherFactoryTest { + private String shardId = "TestShard"; + private RecordsFetcherFactory recordsFetcherFactory; + + @Mock + private GetRecordsRetrievalStrategy getRecordsRetrievalStrategy; + @Mock + private IMetricsFactory metricsFactory; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + recordsFetcherFactory = new SimpleRecordsFetcherFactory(); + } + + @Test + public void createDefaultRecordsFetcherTest() { + GetRecordsCache recordsCache = recordsFetcherFactory.createRecordsFetcher(getRecordsRetrievalStrategy, shardId, + metricsFactory, 1); + assertThat(recordsCache, instanceOf(BlockingGetRecordsCache.class)); + } + + @Test + public void createPrefetchRecordsFetcherTest() { + recordsFetcherFactory.setDataFetchingStrategy(DataFetchingStrategy.PREFETCH_CACHED); + GetRecordsCache recordsCache = recordsFetcherFactory.createRecordsFetcher(getRecordsRetrievalStrategy, shardId, + metricsFactory, 1); + assertThat(recordsCache, instanceOf(PrefetchGetRecordsCache.class)); + } + +} diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/SequenceNumberValidatorTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/SequenceNumberValidatorTest.java index aae93f29..51d1376d 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/SequenceNumberValidatorTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/SequenceNumberValidatorTest.java @@ -87,7 +87,7 @@ public class SequenceNumberValidatorTest { boolean validateWithGetIterator) { String[] nonNumericStrings = { null, "bogus-sequence-number", SentinelCheckpoint.LATEST.toString(), - SentinelCheckpoint.SHARD_END.toString(), SentinelCheckpoint.TRIM_HORIZON.toString(), + SentinelCheckpoint.TRIM_HORIZON.toString(), SentinelCheckpoint.AT_TIMESTAMP.toString() }; for (String nonNumericString : nonNumericStrings) { diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumerTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumerTest.java index 893f64ed..216d59cd 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumerTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumerTest.java @@ -1,16 +1,16 @@ /* - * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; @@ -22,10 +22,13 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyInt; import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.argThat; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -37,14 +40,22 @@ import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.ListIterator; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.hamcrest.Description; +import org.hamcrest.Matcher; +import org.hamcrest.TypeSafeMatcher; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; @@ -52,18 +63,21 @@ import org.mockito.runners.MockitoJUnitRunner; import com.amazonaws.services.kinesis.clientlibrary.interfaces.ICheckpoint; import com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessor; +import com.amazonaws.services.kinesis.clientlibrary.lib.checkpoint.Checkpoint; import com.amazonaws.services.kinesis.clientlibrary.lib.checkpoint.InMemoryCheckpointImpl; import com.amazonaws.services.kinesis.clientlibrary.proxies.IKinesisProxy; import com.amazonaws.services.kinesis.clientlibrary.proxies.KinesisLocalFileProxy; import com.amazonaws.services.kinesis.clientlibrary.proxies.util.KinesisLocalFileDataCreator; import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber; import com.amazonaws.services.kinesis.clientlibrary.types.InitializationInput; +import com.amazonaws.services.kinesis.clientlibrary.types.ShutdownInput; import com.amazonaws.services.kinesis.clientlibrary.types.UserRecord; import com.amazonaws.services.kinesis.leases.impl.KinesisClientLease; import com.amazonaws.services.kinesis.leases.interfaces.ILeaseManager; import com.amazonaws.services.kinesis.metrics.impl.NullMetricsFactory; import com.amazonaws.services.kinesis.metrics.interfaces.IMetricsFactory; import com.amazonaws.services.kinesis.model.Record; +import com.amazonaws.services.kinesis.model.Shard; import com.amazonaws.services.kinesis.model.ShardIteratorType; /** @@ -87,10 +101,17 @@ public class ShardConsumerTest { // Use Executors.newFixedThreadPool since it returns ThreadPoolExecutor, which is // ... a non-final public class, and so can be mocked and spied. private final ExecutorService executorService = Executors.newFixedThreadPool(1); - + private RecordsFetcherFactory recordsFetcherFactory; + + private GetRecordsCache getRecordsCache; + + private KinesisDataFetcher dataFetcher; + @Mock private IRecordProcessor processor; @Mock + private KinesisClientLibConfiguration config; + @Mock private IKinesisProxy streamProxy; @Mock private ILeaseManager leaseManager; @@ -99,6 +120,16 @@ public class ShardConsumerTest { @Mock private ShutdownNotification shutdownNotification; + @Before + public void setup() { + getRecordsCache = null; + dataFetcher = null; + + recordsFetcherFactory = spy(new SimpleRecordsFetcherFactory()); + when(config.getRecordsFetcherFactory()).thenReturn(recordsFetcherFactory); + when(config.getLogWarningForTaskAfterMillis()).thenReturn(Optional.empty()); + } + /** * Test method to verify consumer stays in INITIALIZING state when InitializationTask fails. */ @@ -108,6 +139,7 @@ public class ShardConsumerTest { ShardInfo shardInfo = new ShardInfo("s-0-0", "testToken", null, ExtendedSequenceNumber.TRIM_HORIZON); when(checkpoint.getCheckpoint(anyString())).thenThrow(NullPointerException.class); + when(checkpoint.getCheckpointObject(anyString())).thenThrow(NullPointerException.class); when(leaseManager.getLease(anyString())).thenReturn(null); StreamConfig streamConfig = @@ -128,8 +160,9 @@ public class ShardConsumerTest { executorService, metricsFactory, taskBackoffTimeMillis, - KinesisClientLibConfiguration.DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST); - + KinesisClientLibConfiguration.DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST, + config); + assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); consumer.consumeShard(); // initialize Thread.sleep(50L); @@ -145,7 +178,6 @@ public class ShardConsumerTest { assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.INITIALIZING))); } - /** * Test method to verify consumer stays in INITIALIZING state when InitializationTask fails. */ @@ -156,6 +188,7 @@ public class ShardConsumerTest { ExecutorService spyExecutorService = spy(executorService); when(checkpoint.getCheckpoint(anyString())).thenThrow(NullPointerException.class); + when(checkpoint.getCheckpointObject(anyString())).thenThrow(NullPointerException.class); when(leaseManager.getLease(anyString())).thenReturn(null); StreamConfig streamConfig = new StreamConfig(streamProxy, @@ -175,7 +208,8 @@ public class ShardConsumerTest { spyExecutorService, metricsFactory, taskBackoffTimeMillis, - KinesisClientLibConfiguration.DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST); + KinesisClientLibConfiguration.DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST, + config); assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); consumer.consumeShard(); // initialize @@ -216,10 +250,14 @@ public class ShardConsumerTest { executorService, metricsFactory, taskBackoffTimeMillis, - KinesisClientLibConfiguration.DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST); + KinesisClientLibConfiguration.DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST, + config); + final ExtendedSequenceNumber checkpointSequenceNumber = new ExtendedSequenceNumber("123"); + final ExtendedSequenceNumber pendingCheckpointSequenceNumber = null; when(leaseManager.getLease(anyString())).thenReturn(null); - when(checkpoint.getCheckpoint(anyString())).thenReturn(new ExtendedSequenceNumber("123")); + when(checkpoint.getCheckpointObject(anyString())).thenReturn( + new Checkpoint(checkpointSequenceNumber, pendingCheckpointSequenceNumber)); assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); consumer.consumeShard(); // submit BlockOnParentShardTask @@ -233,7 +271,8 @@ public class ShardConsumerTest { consumer.consumeShard(); // submit InitializeTask Thread.sleep(50L); assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.INITIALIZING))); - verify(processor, times(1)).initialize(any(InitializationInput.class)); + verify(processor, times(1)).initialize(argThat( + initializationInputMatcher(checkpointSequenceNumber, pendingCheckpointSequenceNumber))); try { // Checking the status of submitted InitializeTask from above should throw exception. @@ -244,14 +283,17 @@ public class ShardConsumerTest { } Thread.sleep(50L); assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.INITIALIZING))); - verify(processor, times(1)).initialize(any(InitializationInput.class)); + verify(processor, times(1)).initialize(argThat( + initializationInputMatcher(checkpointSequenceNumber, pendingCheckpointSequenceNumber))); doNothing().when(processor).initialize(any(InitializationInput.class)); consumer.consumeShard(); // submit InitializeTask again. Thread.sleep(50L); assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.INITIALIZING))); - verify(processor, times(2)).initialize(any(InitializationInput.class)); + verify(processor, times(2)).initialize(argThat( + initializationInputMatcher(checkpointSequenceNumber, pendingCheckpointSequenceNumber))); + verify(processor, times(2)).initialize(any(InitializationInput.class)); // no other calls with different args // Checking the status of submitted InitializeTask from above should pass. consumer.consumeShard(); @@ -282,7 +324,6 @@ public class ShardConsumerTest { ICheckpoint checkpoint = new InMemoryCheckpointImpl(startSeqNum.toString()); checkpoint.setCheckpoint(streamShardId, ExtendedSequenceNumber.TRIM_HORIZON, testConcurrencyToken); when(leaseManager.getLease(anyString())).thenReturn(null); - TestStreamlet processor = new TestStreamlet(); StreamConfig streamConfig = @@ -293,18 +334,43 @@ public class ShardConsumerTest { skipCheckpointValidationValue, INITIAL_POSITION_LATEST); ShardInfo shardInfo = new ShardInfo(streamShardId, testConcurrencyToken, null, null); + + RecordProcessorCheckpointer recordProcessorCheckpointer = new RecordProcessorCheckpointer( + shardInfo, + checkpoint, + new SequenceNumberValidator( + streamConfig.getStreamProxy(), + shardInfo.getShardId(), + streamConfig.shouldValidateSequenceNumberBeforeCheckpointing() + ), + metricsFactory + ); + + dataFetcher = new KinesisDataFetcher(streamConfig.getStreamProxy(), shardInfo); + + getRecordsCache = spy(new BlockingGetRecordsCache(maxRecords, + new SynchronousGetRecordsRetrievalStrategy(dataFetcher))); + when(recordsFetcherFactory.createRecordsFetcher(any(GetRecordsRetrievalStrategy.class), anyString(), + any(IMetricsFactory.class), anyInt())) + .thenReturn(getRecordsCache); + ShardConsumer consumer = new ShardConsumer(shardInfo, streamConfig, checkpoint, processor, + recordProcessorCheckpointer, leaseManager, parentShardPollIntervalMillis, cleanupLeasesOfCompletedShards, executorService, metricsFactory, taskBackoffTimeMillis, - KinesisClientLibConfiguration.DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST); + KinesisClientLibConfiguration.DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST, + dataFetcher, + Optional.empty(), + Optional.empty(), + config); assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); consumer.consumeShard(); // check on parent shards @@ -313,6 +379,7 @@ public class ShardConsumerTest { assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.INITIALIZING))); consumer.consumeShard(); // initialize processor.getInitializeLatch().await(5, TimeUnit.SECONDS); + verify(getRecordsCache).start(); // We expect to process all records in numRecs calls for (int i = 0; i < numRecs;) { @@ -325,6 +392,8 @@ public class ShardConsumerTest { } Thread.sleep(50L); } + + verify(getRecordsCache, times(5)).getNextResult(); assertThat(processor.getShutdownReason(), nullValue()); consumer.notifyShutdownRequested(shutdownNotification); @@ -348,6 +417,157 @@ public class ShardConsumerTest { verify(shutdownNotification, atLeastOnce()).shutdownComplete(); assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.SHUTDOWN_COMPLETE))); assertThat(processor.getShutdownReason(), is(equalTo(ShutdownReason.ZOMBIE))); + + verify(getRecordsCache).shutdown(); + + executorService.shutdown(); + executorService.awaitTermination(60, TimeUnit.SECONDS); + + String iterator = fileBasedProxy.getIterator(streamShardId, ShardIteratorType.TRIM_HORIZON.toString()); + List expectedRecords = toUserRecords(fileBasedProxy.get(iterator, numRecs).getRecords()); + verifyConsumedRecords(expectedRecords, processor.getProcessedRecords()); + file.delete(); + } + + private static final class TransientShutdownErrorTestStreamlet extends TestStreamlet { + private final CountDownLatch errorShutdownLatch = new CountDownLatch(1); + + @Override + public void shutdown(ShutdownInput input) { + ShutdownReason reason = input.getShutdownReason(); + if (reason.equals(ShutdownReason.TERMINATE) && errorShutdownLatch.getCount() > 0) { + errorShutdownLatch.countDown(); + throw new RuntimeException("test"); + } else { + super.shutdown(input); + } + } + } + + /** + * Test method for {@link ShardConsumer#consumeShard()} that ensures a transient error thrown from the record + * processor's shutdown method with reason terminate will be retried. + */ + @Test + public final void testConsumeShardWithTransientTerminateError() throws Exception { + int numRecs = 10; + BigInteger startSeqNum = BigInteger.ONE; + String streamShardId = "kinesis-0-0"; + String testConcurrencyToken = "testToken"; + List shardList = KinesisLocalFileDataCreator.createShardList(1, "kinesis-0-", startSeqNum); + // Close the shard so that shutdown is called with reason terminate + shardList.get(0).getSequenceNumberRange().setEndingSequenceNumber( + KinesisLocalFileProxy.MAX_SEQUENCE_NUMBER.subtract(BigInteger.ONE).toString()); + File file = KinesisLocalFileDataCreator.generateTempDataFile(shardList, numRecs, "unitTestSCT002"); + + IKinesisProxy fileBasedProxy = new KinesisLocalFileProxy(file.getAbsolutePath()); + + final int maxRecords = 2; + final int idleTimeMS = 0; // keep unit tests fast + ICheckpoint checkpoint = new InMemoryCheckpointImpl(startSeqNum.toString()); + checkpoint.setCheckpoint(streamShardId, ExtendedSequenceNumber.TRIM_HORIZON, testConcurrencyToken); + when(leaseManager.getLease(anyString())).thenReturn(null); + + TransientShutdownErrorTestStreamlet processor = new TransientShutdownErrorTestStreamlet(); + + StreamConfig streamConfig = + new StreamConfig(fileBasedProxy, + maxRecords, + idleTimeMS, + callProcessRecordsForEmptyRecordList, + skipCheckpointValidationValue, INITIAL_POSITION_LATEST); + + ShardInfo shardInfo = new ShardInfo(streamShardId, testConcurrencyToken, null, null); + + dataFetcher = new KinesisDataFetcher(streamConfig.getStreamProxy(), shardInfo); + + getRecordsCache = spy(new BlockingGetRecordsCache(maxRecords, + new SynchronousGetRecordsRetrievalStrategy(dataFetcher))); + when(recordsFetcherFactory.createRecordsFetcher(any(GetRecordsRetrievalStrategy.class), anyString(), + any(IMetricsFactory.class), anyInt())) + .thenReturn(getRecordsCache); + + RecordProcessorCheckpointer recordProcessorCheckpointer = new RecordProcessorCheckpointer( + shardInfo, + checkpoint, + new SequenceNumberValidator( + streamConfig.getStreamProxy(), + shardInfo.getShardId(), + streamConfig.shouldValidateSequenceNumberBeforeCheckpointing() + ), + metricsFactory + ); + + ShardConsumer consumer = + new ShardConsumer(shardInfo, + streamConfig, + checkpoint, + processor, + recordProcessorCheckpointer, + leaseManager, + parentShardPollIntervalMillis, + cleanupLeasesOfCompletedShards, + executorService, + metricsFactory, + taskBackoffTimeMillis, + KinesisClientLibConfiguration.DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST, + dataFetcher, + Optional.empty(), + Optional.empty(), + config); + + assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); + consumer.consumeShard(); // check on parent shards + Thread.sleep(50L); + consumer.consumeShard(); // start initialization + assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.INITIALIZING))); + consumer.consumeShard(); // initialize + processor.getInitializeLatch().await(5, TimeUnit.SECONDS); + verify(getRecordsCache).start(); + + // We expect to process all records in numRecs calls + for (int i = 0; i < numRecs;) { + boolean newTaskSubmitted = consumer.consumeShard(); + if (newTaskSubmitted) { + LOG.debug("New processing task was submitted, call # " + i); + assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.PROCESSING))); + // CHECKSTYLE:IGNORE ModifiedControlVariable FOR NEXT 1 LINES + i += maxRecords; + } + Thread.sleep(50L); + } + + // Consume shards until shutdown terminate is called and it has thrown an exception + for (int i = 0; i < 100; i++) { + consumer.consumeShard(); + if (processor.errorShutdownLatch.await(50, TimeUnit.MILLISECONDS)) { + break; + } + } + assertEquals(0, processor.errorShutdownLatch.getCount()); + + // Wait for a retry of shutdown terminate that should succeed + for (int i = 0; i < 100; i++) { + consumer.consumeShard(); + if (processor.getShutdownLatch().await(50, TimeUnit.MILLISECONDS)) { + break; + } + } + assertEquals(0, processor.getShutdownLatch().getCount()); + + // Wait for shutdown complete now that terminate shutdown is successful + for (int i = 0; i < 100; i++) { + consumer.consumeShard(); + if (consumer.getCurrentState() == ConsumerStates.ShardConsumerState.SHUTDOWN_COMPLETE) { + break; + } + Thread.sleep(50L); + } + assertThat(consumer.getCurrentState(), equalTo(ConsumerStates.ShardConsumerState.SHUTDOWN_COMPLETE)); + + assertThat(processor.getShutdownReason(), is(equalTo(ShutdownReason.TERMINATE))); + + verify(getRecordsCache).shutdown(); executorService.shutdown(); executorService.awaitTermination(60, TimeUnit.SECONDS); @@ -384,7 +604,6 @@ public class ShardConsumerTest { ICheckpoint checkpoint = new InMemoryCheckpointImpl(startSeqNum.toString()); checkpoint.setCheckpoint(streamShardId, ExtendedSequenceNumber.AT_TIMESTAMP, testConcurrencyToken); when(leaseManager.getLease(anyString())).thenReturn(null); - TestStreamlet processor = new TestStreamlet(); StreamConfig streamConfig = @@ -396,18 +615,43 @@ public class ShardConsumerTest { atTimestamp); ShardInfo shardInfo = new ShardInfo(streamShardId, testConcurrencyToken, null, ExtendedSequenceNumber.TRIM_HORIZON); + + RecordProcessorCheckpointer recordProcessorCheckpointer = new RecordProcessorCheckpointer( + shardInfo, + checkpoint, + new SequenceNumberValidator( + streamConfig.getStreamProxy(), + shardInfo.getShardId(), + streamConfig.shouldValidateSequenceNumberBeforeCheckpointing() + ), + metricsFactory + ); + + dataFetcher = new KinesisDataFetcher(streamConfig.getStreamProxy(), shardInfo); + + getRecordsCache = spy(new BlockingGetRecordsCache(maxRecords, + new SynchronousGetRecordsRetrievalStrategy(dataFetcher))); + when(recordsFetcherFactory.createRecordsFetcher(any(GetRecordsRetrievalStrategy.class), anyString(), + any(IMetricsFactory.class), anyInt())) + .thenReturn(getRecordsCache); + ShardConsumer consumer = new ShardConsumer(shardInfo, streamConfig, checkpoint, processor, + recordProcessorCheckpointer, leaseManager, parentShardPollIntervalMillis, cleanupLeasesOfCompletedShards, executorService, metricsFactory, taskBackoffTimeMillis, - KinesisClientLibConfiguration.DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST); + KinesisClientLibConfiguration.DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST, + dataFetcher, + Optional.empty(), + Optional.empty(), + config); assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); consumer.consumeShard(); // check on parent shards @@ -416,6 +660,8 @@ public class ShardConsumerTest { assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.INITIALIZING))); consumer.consumeShard(); // initialize Thread.sleep(50L); + + verify(getRecordsCache).start(); // We expect to process all records in numRecs calls for (int i = 0; i < numRecs;) { @@ -428,6 +674,8 @@ public class ShardConsumerTest { } Thread.sleep(50L); } + + verify(getRecordsCache, times(4)).getNextResult(); assertThat(processor.getShutdownReason(), nullValue()); consumer.beginShutdown(); @@ -440,13 +688,174 @@ public class ShardConsumerTest { executorService.shutdown(); executorService.awaitTermination(60, TimeUnit.SECONDS); + verify(getRecordsCache).shutdown(); + String iterator = fileBasedProxy.getIterator(streamShardId, timestamp); List expectedRecords = toUserRecords(fileBasedProxy.get(iterator, numRecs).getRecords()); + verifyConsumedRecords(expectedRecords, processor.getProcessedRecords()); assertEquals(4, processor.getProcessedRecords().size()); file.delete(); } + @SuppressWarnings("unchecked") + @Test + public final void testConsumeShardInitializedWithPendingCheckpoint() throws Exception { + ShardInfo shardInfo = new ShardInfo("s-0-0", "testToken", null, ExtendedSequenceNumber.TRIM_HORIZON); + StreamConfig streamConfig = + new StreamConfig(streamProxy, + 1, + 10, + callProcessRecordsForEmptyRecordList, + skipCheckpointValidationValue, INITIAL_POSITION_LATEST); + + ShardConsumer consumer = + new ShardConsumer(shardInfo, + streamConfig, + checkpoint, + processor, + null, + parentShardPollIntervalMillis, + cleanupLeasesOfCompletedShards, + executorService, + metricsFactory, + taskBackoffTimeMillis, + KinesisClientLibConfiguration.DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST, + config); + + GetRecordsCache getRecordsCache = spy(consumer.getGetRecordsCache()); + + final ExtendedSequenceNumber checkpointSequenceNumber = new ExtendedSequenceNumber("123"); + final ExtendedSequenceNumber pendingCheckpointSequenceNumber = new ExtendedSequenceNumber("999"); + when(leaseManager.getLease(anyString())).thenReturn(null); + when(config.getRecordsFetcherFactory()).thenReturn(new SimpleRecordsFetcherFactory()); + when(checkpoint.getCheckpointObject(anyString())).thenReturn( + new Checkpoint(checkpointSequenceNumber, pendingCheckpointSequenceNumber)); + + assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); + consumer.consumeShard(); // submit BlockOnParentShardTask + Thread.sleep(50L); + assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); + verify(processor, times(0)).initialize(any(InitializationInput.class)); + + consumer.consumeShard(); // submit InitializeTask + Thread.sleep(50L); + assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.INITIALIZING))); + verify(processor, times(1)).initialize(argThat( + initializationInputMatcher(checkpointSequenceNumber, pendingCheckpointSequenceNumber))); + verify(processor, times(1)).initialize(any(InitializationInput.class)); // no other calls with different args + + consumer.consumeShard(); + Thread.sleep(50L); + assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.PROCESSING))); + } + + @Test + public void testCreateSynchronousGetRecordsRetrieval() { + ShardInfo shardInfo = new ShardInfo("s-0-0", "testToken", null, ExtendedSequenceNumber.TRIM_HORIZON); + StreamConfig streamConfig = + new StreamConfig(streamProxy, + 1, + 10, + callProcessRecordsForEmptyRecordList, + skipCheckpointValidationValue, INITIAL_POSITION_LATEST); + + ShardConsumer shardConsumer = + new ShardConsumer(shardInfo, + streamConfig, + checkpoint, + processor, + null, + parentShardPollIntervalMillis, + cleanupLeasesOfCompletedShards, + executorService, + metricsFactory, + taskBackoffTimeMillis, + KinesisClientLibConfiguration.DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST, + Optional.empty(), + Optional.empty(), + config); + + assertEquals(shardConsumer.getGetRecordsCache().getGetRecordsRetrievalStrategy().getClass(), + SynchronousGetRecordsRetrievalStrategy.class); + } + + @Test + public void testCreateAsynchronousGetRecordsRetrieval() { + ShardInfo shardInfo = new ShardInfo("s-0-0", "testToken", null, ExtendedSequenceNumber.TRIM_HORIZON); + StreamConfig streamConfig = + new StreamConfig(streamProxy, + 1, + 10, + callProcessRecordsForEmptyRecordList, + skipCheckpointValidationValue, INITIAL_POSITION_LATEST); + + ShardConsumer shardConsumer = + new ShardConsumer(shardInfo, + streamConfig, + checkpoint, + processor, + null, + parentShardPollIntervalMillis, + cleanupLeasesOfCompletedShards, + executorService, + metricsFactory, + taskBackoffTimeMillis, + KinesisClientLibConfiguration.DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST, + Optional.of(1), + Optional.of(2), + config); + + assertEquals(shardConsumer.getGetRecordsCache().getGetRecordsRetrievalStrategy().getClass(), + AsynchronousGetRecordsRetrievalStrategy.class); + } + + @SuppressWarnings("unchecked") + @Test + public void testLongRunningTasks() throws InterruptedException { + final long sleepTime = 1000L; + ExecutorService mockExecutorService = mock(ExecutorService.class); + Future mockFuture = mock(Future.class); + + when(mockExecutorService.submit(any(ITask.class))).thenReturn(mockFuture); + when(mockFuture.isDone()).thenReturn(false); + when(mockFuture.isCancelled()).thenReturn(false); + when(config.getLogWarningForTaskAfterMillis()).thenReturn(Optional.of(sleepTime)); + + ShardInfo shardInfo = new ShardInfo("s-0-0", "testToken", null, ExtendedSequenceNumber.LATEST); + StreamConfig streamConfig = new StreamConfig( + streamProxy, + 1, + 10, + callProcessRecordsForEmptyRecordList, + skipCheckpointValidationValue, + INITIAL_POSITION_LATEST); + + ShardConsumer shardConsumer = new ShardConsumer( + shardInfo, + streamConfig, + checkpoint, + processor, + null, + parentShardPollIntervalMillis, + cleanupLeasesOfCompletedShards, + mockExecutorService, + metricsFactory, + taskBackoffTimeMillis, + KinesisClientLibConfiguration.DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST, + config); + + shardConsumer.consumeShard(); + + Thread.sleep(sleepTime); + + shardConsumer.consumeShard(); + + verify(config).getLogWarningForTaskAfterMillis(); + verify(mockFuture).isDone(); + verify(mockFuture).isCancelled(); + } + //@formatter:off (gets the formatting wrong) private void verifyConsumedRecords(List expectedRecords, List actualRecords) { @@ -469,4 +878,21 @@ public class ShardConsumerTest { } return userRecords; } + + Matcher initializationInputMatcher(final ExtendedSequenceNumber checkpoint, + final ExtendedSequenceNumber pendingCheckpoint) { + return new TypeSafeMatcher() { + @Override + protected boolean matchesSafely(InitializationInput item) { + return Objects.equals(checkpoint, item.getExtendedSequenceNumber()) + && Objects.equals(pendingCheckpoint, item.getPendingCheckpointSequenceNumber()); + } + + @Override + public void describeTo(Description description) { + description.appendText(String.format("Checkpoint should be %s and pending checkpoint should be %s", + checkpoint, pendingCheckpoint)); + } + }; + } } diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTaskIntegrationTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTaskIntegrationTest.java index 307596e3..619f3eaf 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTaskIntegrationTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTaskIntegrationTest.java @@ -90,6 +90,7 @@ public class ShardSyncTaskIntegrationTest { new KinesisClientLeaseManager("ShardSyncTaskIntegrationTest", new AmazonDynamoDBClient(credentialsProvider), useConsistentReads); + kinesisProxy = new KinesisProxy(STREAM_NAME, new DefaultAWSCredentialsProviderChain(), @@ -106,7 +107,6 @@ public class ShardSyncTaskIntegrationTest { /** * Test method for call(). * - * @throws CapacityExceededException * @throws DependencyException * @throws InvalidStateException * @throws ProvisionedThroughputException @@ -124,6 +124,7 @@ public class ShardSyncTaskIntegrationTest { leaseManager, InitialPositionInStreamExtended.newInitialPosition(InitialPositionInStream.LATEST), false, + false, 0L); syncTask.call(); List leases = leaseManager.listLeases(); diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncerTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncerTest.java index b8f6ae56..7ff12542 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncerTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncerTest.java @@ -146,6 +146,39 @@ public class ShardSyncerTest { } } + /** + * Test determineNewLeasesToCreate() where there are no leases and no resharding operations have been performed, but one of + * the shards was marked as inconsistent. + */ + @Test + public final void testDetermineNewLeasesToCreate0Leases0Reshards1Inconsistent() { + List shards = new ArrayList(); + List currentLeases = new ArrayList(); + SequenceNumberRange sequenceRange = ShardObjectHelper.newSequenceNumberRange("342980", null); + + String shardId0 = "shardId-0"; + shards.add(ShardObjectHelper.newShard(shardId0, null, null, sequenceRange)); + + String shardId1 = "shardId-1"; + shards.add(ShardObjectHelper.newShard(shardId1, null, null, sequenceRange)); + + String shardId2 = "shardId-2"; + shards.add(ShardObjectHelper.newShard(shardId2, shardId1, null, sequenceRange)); + + Set inconsistentShardIds = new HashSet(); + inconsistentShardIds.add(shardId2); + + List newLeases = + ShardSyncer.determineNewLeasesToCreate(shards, currentLeases, INITIAL_POSITION_LATEST, inconsistentShardIds); + Assert.assertEquals(2, newLeases.size()); + Set expectedLeaseShardIds = new HashSet(); + expectedLeaseShardIds.add(shardId0); + expectedLeaseShardIds.add(shardId1); + for (KinesisClientLease lease : newLeases) { + Assert.assertTrue(expectedLeaseShardIds.contains(lease.getLeaseKey())); + } + } + /** * Test bootstrapShardLeases() starting at TRIM_HORIZON ("beginning" of stream) * @@ -296,6 +329,41 @@ public class ShardSyncerTest { dataFile.delete(); } + /** + * Test checkAndCreateLeasesForNewShards() when a parent is open and children of open parents are being ignored. + */ + @Test + public final void testCheckAndCreateLeasesForNewShardsWhenParentIsOpenAndIgnoringInconsistentChildren() + throws KinesisClientLibIOException, DependencyException, InvalidStateException, ProvisionedThroughputException, + IOException { + List shards = constructShardListForGraphA(); + Shard shard = shards.get(5); + Assert.assertEquals("shardId-5", shard.getShardId()); + SequenceNumberRange range = shard.getSequenceNumberRange(); + // shardId-5 in graph A has two children (shardId-9 and shardId-10). if shardId-5 + // is not closed, those children should be ignored when syncing shards, no leases + // should be obtained for them, and we should obtain a lease on the still-open + // parent. + range.setEndingSequenceNumber(null); + shard.setSequenceNumberRange(range); + File dataFile = KinesisLocalFileDataCreator.generateTempDataFile(shards, 2, "testBootstrap1"); + dataFile.deleteOnExit(); + IKinesisProxy kinesisProxy = new KinesisLocalFileProxy(dataFile.getAbsolutePath()); + ShardSyncer.checkAndCreateLeasesForNewShards(kinesisProxy, leaseManager, INITIAL_POSITION_LATEST, + cleanupLeasesOfCompletedShards, true); + List newLeases = leaseManager.listLeases(); + Set expectedLeaseShardIds = new HashSet(); + expectedLeaseShardIds.add("shardId-4"); + expectedLeaseShardIds.add("shardId-5"); + expectedLeaseShardIds.add("shardId-8"); + Assert.assertEquals(expectedLeaseShardIds.size(), newLeases.size()); + for (KinesisClientLease lease1 : newLeases) { + Assert.assertTrue(expectedLeaseShardIds.contains(lease1.getLeaseKey())); + Assert.assertEquals(ExtendedSequenceNumber.LATEST, lease1.getCheckpoint()); + } + dataFile.delete(); + } + /** * @throws KinesisClientLibIOException * @throws DependencyException @@ -586,7 +654,8 @@ public class ShardSyncerTest { dataFile.deleteOnExit(); IKinesisProxy kinesisProxy = new KinesisLocalFileProxy(dataFile.getAbsolutePath()); - ShardSyncer.bootstrapShardLeases(kinesisProxy, leaseManager, initialPosition, cleanupLeasesOfCompletedShards); + ShardSyncer.bootstrapShardLeases(kinesisProxy, leaseManager, initialPosition, cleanupLeasesOfCompletedShards, + false); List newLeases = leaseManager.listLeases(); Assert.assertEquals(2, newLeases.size()); Set expectedLeaseShardIds = new HashSet(); @@ -955,9 +1024,9 @@ public class ShardSyncerTest { SequenceNumberRange range0 = ShardObjectHelper.newSequenceNumberRange("11", "102"); SequenceNumberRange range1 = ShardObjectHelper.newSequenceNumberRange("11", null); - SequenceNumberRange range2 = ShardObjectHelper.newSequenceNumberRange("11", "205"); - SequenceNumberRange range3 = ShardObjectHelper.newSequenceNumberRange("103", "205"); - SequenceNumberRange range4 = ShardObjectHelper.newSequenceNumberRange("206", null); + SequenceNumberRange range2 = ShardObjectHelper.newSequenceNumberRange("11", "210"); + SequenceNumberRange range3 = ShardObjectHelper.newSequenceNumberRange("103", "210"); + SequenceNumberRange range4 = ShardObjectHelper.newSequenceNumberRange("211", null); HashKeyRange hashRange0 = ShardObjectHelper.newHashKeyRange("0", "99"); HashKeyRange hashRange1 = ShardObjectHelper.newHashKeyRange("100", "199"); diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownFutureTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownFutureTest.java deleted file mode 100644 index cccbc9a1..00000000 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownFutureTest.java +++ /dev/null @@ -1,236 +0,0 @@ -package com.amazonaws.services.kinesis.clientlibrary.lib.worker; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyLong; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.runners.MockitoJUnitRunner; -import org.mockito.stubbing.Answer; -import org.mockito.stubbing.OngoingStubbing; - -@RunWith(MockitoJUnitRunner.class) -public class ShutdownFutureTest { - - @Mock - private CountDownLatch shutdownCompleteLatch; - @Mock - private CountDownLatch notificationCompleteLatch; - @Mock - private Worker worker; - @Mock - private ConcurrentMap shardInfoConsumerMap; - - @Test - public void testSimpleGetAlreadyCompleted() throws Exception { - ShutdownFuture future = new ShutdownFuture(shutdownCompleteLatch, notificationCompleteLatch, worker); - - mockNotificationComplete(true); - mockShutdownComplete(true); - - future.get(); - - verify(notificationCompleteLatch).await(anyLong(), any(TimeUnit.class)); - verify(worker).shutdown(); - verify(shutdownCompleteLatch).await(anyLong(), any(TimeUnit.class)); - } - - @Test - public void testNotificationNotCompleted() throws Exception { - ShutdownFuture future = new ShutdownFuture(shutdownCompleteLatch, notificationCompleteLatch, worker); - - mockNotificationComplete(false, true); - mockShutdownComplete(true); - - when(worker.getShardInfoShardConsumerMap()).thenReturn(shardInfoConsumerMap); - when(shardInfoConsumerMap.isEmpty()).thenReturn(false); - when(worker.isShutdownComplete()).thenReturn(false); - - when(notificationCompleteLatch.getCount()).thenReturn(1L); - when(shutdownCompleteLatch.getCount()).thenReturn(1L); - - expectedTimeoutException(future); - - verify(worker, never()).shutdown(); - - awaitFuture(future); - - verify(notificationCompleteLatch).getCount(); - verifyLatchAwait(notificationCompleteLatch, 2); - - verify(shutdownCompleteLatch).getCount(); - verifyLatchAwait(shutdownCompleteLatch); - - verify(worker).shutdown(); - - } - - @Test - public void testShutdownNotCompleted() throws Exception { - ShutdownFuture future = new ShutdownFuture(shutdownCompleteLatch, notificationCompleteLatch, worker); - mockNotificationComplete(true); - mockShutdownComplete(false, true); - - when(shutdownCompleteLatch.getCount()).thenReturn(1L); - when(worker.isShutdownComplete()).thenReturn(false); - - mockShardInfoConsumerMap(1); - - expectedTimeoutException(future); - verify(worker).shutdown(); - awaitFuture(future); - - verifyLatchAwait(notificationCompleteLatch, 2); - verifyLatchAwait(shutdownCompleteLatch, 2); - - verify(worker).isShutdownComplete(); - verify(worker).getShardInfoShardConsumerMap(); - - } - - @Test - public void testShutdownNotCompleteButWorkerShutdown() throws Exception { - ShutdownFuture future = create(); - - mockNotificationComplete(true); - mockShutdownComplete(false); - - when(shutdownCompleteLatch.getCount()).thenReturn(1L); - when(worker.isShutdownComplete()).thenReturn(true); - mockShardInfoConsumerMap(1); - - awaitFuture(future); - verify(worker).shutdown(); - verifyLatchAwait(notificationCompleteLatch); - verifyLatchAwait(shutdownCompleteLatch); - - verify(worker, times(2)).isShutdownComplete(); - verify(worker).getShardInfoShardConsumerMap(); - verify(shardInfoConsumerMap).size(); - } - - @Test - public void testShutdownNotCompleteButShardConsumerEmpty() throws Exception { - ShutdownFuture future = create(); - mockNotificationComplete(true); - mockShutdownComplete(false); - - mockOutstanding(shutdownCompleteLatch, 1L); - - when(worker.isShutdownComplete()).thenReturn(false); - mockShardInfoConsumerMap(0); - - awaitFuture(future); - verify(worker).shutdown(); - verifyLatchAwait(notificationCompleteLatch); - verifyLatchAwait(shutdownCompleteLatch); - - verify(worker, times(2)).isShutdownComplete(); - verify(worker, times(2)).getShardInfoShardConsumerMap(); - - verify(shardInfoConsumerMap).isEmpty(); - verify(shardInfoConsumerMap).size(); - } - - @Test - public void testNotificationNotCompleteButShardConsumerEmpty() throws Exception { - ShutdownFuture future = create(); - mockNotificationComplete(false); - mockShutdownComplete(false); - - mockOutstanding(notificationCompleteLatch, 1L); - mockOutstanding(shutdownCompleteLatch, 1L); - - when(worker.isShutdownComplete()).thenReturn(false); - mockShardInfoConsumerMap(0); - - awaitFuture(future); - verify(worker, never()).shutdown(); - verifyLatchAwait(notificationCompleteLatch); - verify(shutdownCompleteLatch, never()).await(); - - verify(worker, times(2)).isShutdownComplete(); - verify(worker, times(2)).getShardInfoShardConsumerMap(); - - verify(shardInfoConsumerMap).isEmpty(); - verify(shardInfoConsumerMap).size(); - } - - @Test(expected = TimeoutException.class) - public void testTimeExceededException() throws Exception { - ShutdownFuture future = create(); - mockNotificationComplete(false); - mockOutstanding(notificationCompleteLatch, 1L); - when(worker.isShutdownComplete()).thenReturn(false); - mockShardInfoConsumerMap(1); - - future.get(1, TimeUnit.NANOSECONDS); - } - - private ShutdownFuture create() { - return new ShutdownFuture(shutdownCompleteLatch, notificationCompleteLatch, worker); - } - - private void mockShardInfoConsumerMap(Integer initialItemCount, Integer ... additionalItemCounts) { - when(worker.getShardInfoShardConsumerMap()).thenReturn(shardInfoConsumerMap); - Boolean additionalEmptyStates[] = new Boolean[additionalItemCounts.length]; - for(int i = 0; i < additionalItemCounts.length; ++i) { - additionalEmptyStates[i] = additionalItemCounts[i] == 0; - } - when(shardInfoConsumerMap.size()).thenReturn(initialItemCount, additionalItemCounts); - when(shardInfoConsumerMap.isEmpty()).thenReturn(initialItemCount == 0, additionalEmptyStates); - } - - private void verifyLatchAwait(CountDownLatch latch) throws Exception { - verifyLatchAwait(latch, 1); - } - - private void verifyLatchAwait(CountDownLatch latch, int times) throws Exception { - verify(latch, times(times)).await(anyLong(), any(TimeUnit.class)); - } - - private void expectedTimeoutException(ShutdownFuture future) throws Exception { - boolean gotTimeout = false; - try { - awaitFuture(future); - } catch (TimeoutException te) { - gotTimeout = true; - } - assertThat("Expected a timeout exception to occur", gotTimeout); - } - - private void awaitFuture(ShutdownFuture future) throws Exception { - future.get(1, TimeUnit.SECONDS); - } - - private void mockNotificationComplete(Boolean initial, Boolean... states) throws Exception { - mockLatch(notificationCompleteLatch, initial, states); - - } - - private void mockShutdownComplete(Boolean initial, Boolean... states) throws Exception { - mockLatch(shutdownCompleteLatch, initial, states); - } - - private void mockLatch(CountDownLatch latch, Boolean initial, Boolean... states) throws Exception { - when(latch.await(anyLong(), any(TimeUnit.class))).thenReturn(initial, states); - } - - private void mockOutstanding(CountDownLatch latch, Long remaining, Long ... additionalRemaining) throws Exception { - when(latch.getCount()).thenReturn(remaining, additionalRemaining); - } - -} \ No newline at end of file diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTaskTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTaskTest.java index 9eaf7e8e..ddf07e10 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTaskTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTaskTest.java @@ -1,20 +1,22 @@ /* - * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; +import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.util.HashSet; @@ -34,10 +36,14 @@ import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber import com.amazonaws.services.kinesis.leases.impl.KinesisClientLease; import com.amazonaws.services.kinesis.leases.impl.KinesisClientLeaseManager; import com.amazonaws.services.kinesis.leases.interfaces.ILeaseManager; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; /** * */ +@RunWith(MockitoJUnitRunner.class) public class ShutdownTaskTest { private static final long TASK_BACKOFF_TIME_MILLIS = 1L; private static final InitialPositionInStreamExtended INITIAL_POSITION_TRIM_HORIZON = @@ -51,6 +57,9 @@ public class ShutdownTaskTest { defaultParentShardIds, ExtendedSequenceNumber.LATEST); IRecordProcessor defaultRecordProcessor = new TestStreamlet(); + + @Mock + private GetRecordsCache getRecordsCache; /** * @throws java.lang.Exception @@ -71,6 +80,7 @@ public class ShutdownTaskTest { */ @Before public void setUp() throws Exception { + doNothing().when(getRecordsCache).shutdown(); } /** @@ -90,6 +100,7 @@ public class ShutdownTaskTest { IKinesisProxy kinesisProxy = mock(IKinesisProxy.class); ILeaseManager leaseManager = mock(KinesisClientLeaseManager.class); boolean cleanupLeasesOfCompletedShards = false; + boolean ignoreUnexpectedChildShards = false; ShutdownTask task = new ShutdownTask(defaultShardInfo, defaultRecordProcessor, checkpointer, @@ -97,8 +108,10 @@ public class ShutdownTaskTest { kinesisProxy, INITIAL_POSITION_TRIM_HORIZON, cleanupLeasesOfCompletedShards, + ignoreUnexpectedChildShards, leaseManager, - TASK_BACKOFF_TIME_MILLIS); + TASK_BACKOFF_TIME_MILLIS, + getRecordsCache); TaskResult result = task.call(); Assert.assertNotNull(result.getException()); Assert.assertTrue(result.getException() instanceof IllegalArgumentException); @@ -115,6 +128,7 @@ public class ShutdownTaskTest { when(kinesisProxy.getShardList()).thenReturn(null); ILeaseManager leaseManager = mock(KinesisClientLeaseManager.class); boolean cleanupLeasesOfCompletedShards = false; + boolean ignoreUnexpectedChildShards = false; ShutdownTask task = new ShutdownTask(defaultShardInfo, defaultRecordProcessor, checkpointer, @@ -122,11 +136,14 @@ public class ShutdownTaskTest { kinesisProxy, INITIAL_POSITION_TRIM_HORIZON, cleanupLeasesOfCompletedShards, + ignoreUnexpectedChildShards, leaseManager, - TASK_BACKOFF_TIME_MILLIS); + TASK_BACKOFF_TIME_MILLIS, + getRecordsCache); TaskResult result = task.call(); Assert.assertNotNull(result.getException()); Assert.assertTrue(result.getException() instanceof KinesisClientLibIOException); + verify(getRecordsCache).shutdown(); } /** @@ -134,7 +151,7 @@ public class ShutdownTaskTest { */ @Test public final void testGetTaskType() { - ShutdownTask task = new ShutdownTask(null, null, null, null, null, null, false, null, 0); + ShutdownTask task = new ShutdownTask(null, null, null, null, null, null, false, false, null, 0, getRecordsCache); Assert.assertEquals(TaskType.SHUTDOWN, task.getTaskType()); } diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ThrottlingReporterTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ThrottlingReporterTest.java index d0645229..79118ac9 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ThrottlingReporterTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ThrottlingReporterTest.java @@ -1,3 +1,17 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; import static org.mockito.Matchers.any; diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/WorkerTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/WorkerTest.java index daf58165..ddc39aed 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/WorkerTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/WorkerTest.java @@ -1,16 +1,16 @@ /* - * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; @@ -19,12 +19,26 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.isA; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.argThat; import static org.mockito.Matchers.eq; import static org.mockito.Matchers.same; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.io.File; import java.lang.Thread.State; @@ -60,16 +74,27 @@ import org.hamcrest.Matcher; import org.hamcrest.TypeSafeDiagnosingMatcher; import org.hamcrest.TypeSafeMatcher; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; import org.mockito.Matchers; import org.mockito.Mock; import org.mockito.invocation.InvocationOnMock; import org.mockito.runners.MockitoJUnitRunner; import org.mockito.stubbing.Answer; +import com.amazonaws.ClientConfiguration; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.regions.RegionUtils; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.cloudwatch.AmazonCloudWatch; +import com.amazonaws.services.cloudwatch.AmazonCloudWatchClientBuilder; import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; +import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder; import com.amazonaws.services.dynamodbv2.local.embedded.DynamoDBEmbedded; +import com.amazonaws.services.kinesis.AmazonKinesis; +import com.amazonaws.services.kinesis.AmazonKinesisClientBuilder; import com.amazonaws.services.kinesis.clientlibrary.exceptions.KinesisClientLibNonRetryableException; import com.amazonaws.services.kinesis.clientlibrary.interfaces.ICheckpoint; import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer; @@ -77,8 +102,10 @@ import com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcess import com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessorFactory; import com.amazonaws.services.kinesis.clientlibrary.lib.worker.Worker.WorkerCWMetricsFactory; import com.amazonaws.services.kinesis.clientlibrary.lib.worker.Worker.WorkerThreadPoolExecutor; +import com.amazonaws.services.kinesis.clientlibrary.lib.worker.WorkerStateChangeListener.WorkerState; import com.amazonaws.services.kinesis.clientlibrary.proxies.IKinesisProxy; import com.amazonaws.services.kinesis.clientlibrary.proxies.KinesisLocalFileProxy; +import com.amazonaws.services.kinesis.clientlibrary.proxies.KinesisProxy; import com.amazonaws.services.kinesis.clientlibrary.proxies.util.KinesisLocalFileDataCreator; import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber; import com.amazonaws.services.kinesis.clientlibrary.types.InitializationInput; @@ -130,6 +157,9 @@ public class WorkerTest { private static final String KINESIS_SHARD_ID_FORMAT = "kinesis-0-0-%d"; private static final String CONCURRENCY_TOKEN_FORMAT = "testToken-%d"; + private RecordsFetcherFactory recordsFetcherFactory; + private KinesisClientLibConfiguration config; + @Mock private KinesisClientLibLeaseCoordinator leaseCoordinator; @Mock @@ -154,9 +184,18 @@ public class WorkerTest { private Future taskFuture; @Mock private TaskResult taskResult; + @Mock + private WorkerStateChangeListener workerStateChangeListener; + + @Before + public void setup() { + config = spy(new KinesisClientLibConfiguration("app", null, null, null)); + recordsFetcherFactory = spy(new SimpleRecordsFetcherFactory()); + when(config.getRecordsFetcherFactory()).thenReturn(recordsFetcherFactory); + } // CHECKSTYLE:IGNORE AnonInnerLengthCheck FOR NEXT 50 LINES - private static final com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorFactory SAMPLE_RECORD_PROCESSOR_FACTORY = + private static final com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorFactory SAMPLE_RECORD_PROCESSOR_FACTORY = new com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorFactory() { @Override @@ -189,20 +228,19 @@ public class WorkerTest { }; } }; - - private static final IRecordProcessorFactory SAMPLE_RECORD_PROCESSOR_FACTORY_V2 = + + private static final IRecordProcessorFactory SAMPLE_RECORD_PROCESSOR_FACTORY_V2 = new V1ToV2RecordProcessorFactoryAdapter(SAMPLE_RECORD_PROCESSOR_FACTORY); /** - * Test method for {@link com.amazonaws.services.kinesis.clientlibrary.lib.worker.Worker#getApplicationName()}. + * Test method for {@link Worker#getApplicationName()}. */ @Test public final void testGetStageName() { final String stageName = "testStageName"; - final KinesisClientLibConfiguration clientConfig = - new KinesisClientLibConfiguration(stageName, null, null, null); - Worker worker = new Worker(v1RecordProcessorFactory, clientConfig); + config = new KinesisClientLibConfiguration(stageName, null, null, null); + Worker worker = new Worker(v1RecordProcessorFactory, config); Assert.assertEquals(stageName, worker.getApplicationName()); } @@ -210,6 +248,7 @@ public class WorkerTest { public final void testCreateOrGetShardConsumer() { final String stageName = "testStageName"; IRecordProcessorFactory streamletFactory = SAMPLE_RECORD_PROCESSOR_FACTORY_V2; + config = new KinesisClientLibConfiguration(stageName, null, null, null); IKinesisProxy proxy = null; ICheckpoint checkpoint = null; int maxRecords = 1; @@ -228,7 +267,9 @@ public class WorkerTest { Worker worker = new Worker(stageName, - streamletFactory, streamConfig, INITIAL_POSITION_LATEST, + streamletFactory, + config, + streamConfig, INITIAL_POSITION_LATEST, parentShardPollIntervalMillis, shardSyncIntervalMillis, cleanupLeasesUponShardCompletion, @@ -275,10 +316,22 @@ public class WorkerTest { when(leaseCoordinator.getCurrentAssignments()).thenReturn(initialState).thenReturn(firstCheckpoint) .thenReturn(secondCheckpoint); - Worker worker = new Worker(stageName, streamletFactory, streamConfig, INITIAL_POSITION_LATEST, - parentShardPollIntervalMillis, shardSyncIntervalMillis, cleanupLeasesUponShardCompletion, checkpoint, - leaseCoordinator, execService, nullMetricsFactory, taskBackoffTimeMillis, failoverTimeMillis, - KinesisClientLibConfiguration.DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST, shardPrioritization); + Worker worker = new Worker(stageName, + streamletFactory, + config, + streamConfig, + INITIAL_POSITION_LATEST, + parentShardPollIntervalMillis, + shardSyncIntervalMillis, + cleanupLeasesUponShardCompletion, + checkpoint, + leaseCoordinator, + execService, + nullMetricsFactory, + taskBackoffTimeMillis, + failoverTimeMillis, + KinesisClientLibConfiguration.DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST, + shardPrioritization); Worker workerSpy = spy(worker); @@ -314,6 +367,7 @@ public class WorkerTest { public final void testCleanupShardConsumers() { final String stageName = "testStageName"; IRecordProcessorFactory streamletFactory = SAMPLE_RECORD_PROCESSOR_FACTORY_V2; + config = new KinesisClientLibConfiguration(stageName, null, null, null); IKinesisProxy proxy = null; ICheckpoint checkpoint = null; int maxRecords = 1; @@ -332,7 +386,9 @@ public class WorkerTest { Worker worker = new Worker(stageName, - streamletFactory, streamConfig, INITIAL_POSITION_LATEST, + streamletFactory, + config, + streamConfig, INITIAL_POSITION_LATEST, parentShardPollIntervalMillis, shardSyncIntervalMillis, cleanupLeasesUponShardCompletion, @@ -371,6 +427,7 @@ public class WorkerTest { public final void testInitializationFailureWithRetries() { String stageName = "testInitializationWorker"; IRecordProcessorFactory recordProcessorFactory = new TestStreamletFactory(null, null); + config = new KinesisClientLibConfiguration(stageName, null, null, null); int count = 0; when(proxy.getShardList()).thenThrow(new RuntimeException(Integer.toString(count++))); int maxRecords = 2; @@ -386,6 +443,7 @@ public class WorkerTest { Worker worker = new Worker(stageName, recordProcessorFactory, + config, streamConfig, INITIAL_POSITION_TRIM_HORIZON, shardPollInterval, shardSyncIntervalMillis, @@ -437,7 +495,7 @@ public class WorkerTest { /** * Runs worker with threadPoolSize < numShards - * Test method for {@link com.amazonaws.services.kinesis.clientlibrary.lib.worker.Worker#run()}. + * Test method for {@link Worker#run()}. */ @Test public final void testOneSplitShard2Threads() throws Exception { @@ -448,12 +506,12 @@ public class WorkerTest { KinesisClientLease lease = ShardSyncer.newKCLLease(shardList.get(0)); lease.setCheckpoint(new ExtendedSequenceNumber("2")); initialLeases.add(lease); - runAndTestWorker(shardList, threadPoolSize, initialLeases, callProcessRecordsForEmptyRecordList, numberOfRecordsPerShard); + runAndTestWorker(shardList, threadPoolSize, initialLeases, callProcessRecordsForEmptyRecordList, numberOfRecordsPerShard, config); } /** * Runs worker with threadPoolSize < numShards - * Test method for {@link com.amazonaws.services.kinesis.clientlibrary.lib.worker.Worker#run()}. + * Test method for {@link Worker#run()}. */ @Test public final void testOneSplitShard2ThreadsWithCallsForEmptyRecords() throws Exception { @@ -465,7 +523,10 @@ public class WorkerTest { lease.setCheckpoint(new ExtendedSequenceNumber("2")); initialLeases.add(lease); boolean callProcessRecordsForEmptyRecordList = true; - runAndTestWorker(shardList, threadPoolSize, initialLeases, callProcessRecordsForEmptyRecordList, numberOfRecordsPerShard); + RecordsFetcherFactory recordsFetcherFactory = new SimpleRecordsFetcherFactory(); + recordsFetcherFactory.setIdleMillisBetweenCalls(0L); + when(config.getRecordsFetcherFactory()).thenReturn(recordsFetcherFactory); + runAndTestWorker(shardList, threadPoolSize, initialLeases, callProcessRecordsForEmptyRecordList, numberOfRecordsPerShard, config); } @Test @@ -490,7 +551,8 @@ public class WorkerTest { 10, kinesisProxy, v2RecordProcessorFactory, executorService, - cwMetricsFactory); + cwMetricsFactory, + config); // Give some time for thread to run. workerStarted.await(); @@ -526,7 +588,8 @@ public class WorkerTest { 10, kinesisProxy, v2RecordProcessorFactory, executorService, - cwMetricsFactory); + cwMetricsFactory, + config); // Give some time for thread to run. workerStarted.await(); @@ -573,6 +636,14 @@ public class WorkerTest { } }).when(v2RecordProcessor).processRecords(any(ProcessRecordsInput.class)); + RecordsFetcherFactory recordsFetcherFactory = mock(RecordsFetcherFactory.class); + GetRecordsCache getRecordsCache = mock(GetRecordsCache.class); + when(config.getRecordsFetcherFactory()).thenReturn(recordsFetcherFactory); + when(recordsFetcherFactory.createRecordsFetcher(any(GetRecordsRetrievalStrategy.class), anyString(), + any(IMetricsFactory.class), anyInt())) + .thenReturn(getRecordsCache); + when(getRecordsCache.getNextResult()).thenReturn(new ProcessRecordsInput().withRecords(Collections.emptyList()).withMillisBehindLatest(0L)); + WorkerThread workerThread = runWorker(shardList, initialLeases, callProcessRecordsForEmptyRecordList, @@ -581,7 +652,8 @@ public class WorkerTest { fileBasedProxy, v2RecordProcessorFactory, executorService, - nullMetricsFactory); + nullMetricsFactory, + config); // Only sleep for time that is required. processRecordsLatch.await(); @@ -603,7 +675,7 @@ public class WorkerTest { * This test is testing the {@link Worker}'s shutdown behavior and by extension the behavior of * {@link ThreadPoolExecutor#shutdownNow()}. It depends on the thread pool sending an interrupt to the pool threads. * This behavior makes the test a bit racy, since we need to ensure a specific order of events. - * + * * @throws Exception */ @Test @@ -672,7 +744,8 @@ public class WorkerTest { fileBasedProxy, v2RecordProcessorFactory, executorService, - nullMetricsFactory); + nullMetricsFactory, + config); // Only sleep for time that is required. processRecordsLatch.await(); @@ -742,10 +815,22 @@ public class WorkerTest { when(recordProcessorFactory.createProcessor()).thenReturn(processor); - Worker worker = new Worker("testRequestShutdown", recordProcessorFactory, streamConfig, - INITIAL_POSITION_TRIM_HORIZON, parentShardPollIntervalMillis, shardSyncIntervalMillis, - cleanupLeasesUponShardCompletion, leaseCoordinator, leaseCoordinator, executorService, metricsFactory, - taskBackoffTimeMillis, failoverTimeMillis, false, shardPrioritization); + Worker worker = new Worker("testRequestShutdown", + recordProcessorFactory, + config, + streamConfig, + INITIAL_POSITION_TRIM_HORIZON, + parentShardPollIntervalMillis, + shardSyncIntervalMillis, + cleanupLeasesUponShardCompletion, + leaseCoordinator, + leaseCoordinator, + executorService, + metricsFactory, + taskBackoffTimeMillis, + failoverTimeMillis, + false, + shardPrioritization); when(executorService.submit(Matchers.> any())) .thenAnswer(new ShutdownHandlingAnswer(taskFuture)); @@ -762,7 +847,7 @@ public class WorkerTest { verify(executorService, atLeastOnce()).submit(argThat( both(isA(MetricsCollectingTaskDecorator.class)).and(TaskTypeMatcher.isOfType(TaskType.INITIALIZE)))); - worker.requestShutdown(); + worker.createWorkerShutdownCallable().call(); worker.runProcessLoop(); verify(executorService, atLeastOnce()).submit(argThat(both(isA(MetricsCollectingTaskDecorator.class)) @@ -781,6 +866,158 @@ public class WorkerTest { } + @Test(expected = IllegalStateException.class) + public void testShutdownCallableNotAllowedTwice() throws Exception { + + IRecordProcessorFactory recordProcessorFactory = mock(IRecordProcessorFactory.class); + StreamConfig streamConfig = mock(StreamConfig.class); + IMetricsFactory metricsFactory = mock(IMetricsFactory.class); + + ExtendedSequenceNumber checkpoint = new ExtendedSequenceNumber("123", 0L); + KinesisClientLeaseBuilder builder = new KinesisClientLeaseBuilder().withCheckpoint(checkpoint) + .withConcurrencyToken(UUID.randomUUID()).withLastCounterIncrementNanos(0L).withLeaseCounter(0L) + .withOwnerSwitchesSinceCheckpoint(0L).withLeaseOwner("Self"); + + final List leases = new ArrayList<>(); + final List currentAssignments = new ArrayList<>(); + KinesisClientLease lease = builder.withLeaseKey(String.format("shardId-%03d", 1)).build(); + leases.add(lease); + currentAssignments.add(new ShardInfo(lease.getLeaseKey(), lease.getConcurrencyToken().toString(), + lease.getParentShardIds(), lease.getCheckpoint())); + + when(leaseCoordinator.getAssignments()).thenAnswer(new Answer>() { + @Override + public List answer(InvocationOnMock invocation) throws Throwable { + return leases; + } + }); + when(leaseCoordinator.getCurrentAssignments()).thenAnswer(new Answer>() { + @Override + public List answer(InvocationOnMock invocation) throws Throwable { + return currentAssignments; + } + }); + + IRecordProcessor processor = mock(IRecordProcessor.class); + when(recordProcessorFactory.createProcessor()).thenReturn(processor); + + Worker worker = new InjectableWorker("testRequestShutdown", recordProcessorFactory, config, streamConfig, + INITIAL_POSITION_TRIM_HORIZON, parentShardPollIntervalMillis, shardSyncIntervalMillis, + cleanupLeasesUponShardCompletion, leaseCoordinator, leaseCoordinator, executorService, metricsFactory, + taskBackoffTimeMillis, failoverTimeMillis, false, shardPrioritization) { + @Override + void postConstruct() { + this.gracefuleShutdownStarted = true; + } + }; + + when(executorService.submit(Matchers.> any())) + .thenAnswer(new ShutdownHandlingAnswer(taskFuture)); + when(taskFuture.isDone()).thenReturn(true); + when(taskFuture.get()).thenReturn(taskResult); + + worker.runProcessLoop(); + + verify(executorService, atLeastOnce()).submit(argThat(both(isA(MetricsCollectingTaskDecorator.class)) + .and(TaskTypeMatcher.isOfType(TaskType.BLOCK_ON_PARENT_SHARDS)))); + + worker.runProcessLoop(); + + verify(executorService, atLeastOnce()).submit(argThat( + both(isA(MetricsCollectingTaskDecorator.class)).and(TaskTypeMatcher.isOfType(TaskType.INITIALIZE)))); + + assertThat(worker.hasGracefulShutdownStarted(), equalTo(true)); + worker.createWorkerShutdownCallable().call(); + + } + + @Test + public void testGracefulShutdownSingleFuture() throws Exception { + + IRecordProcessorFactory recordProcessorFactory = mock(IRecordProcessorFactory.class); + StreamConfig streamConfig = mock(StreamConfig.class); + IMetricsFactory metricsFactory = mock(IMetricsFactory.class); + + ExtendedSequenceNumber checkpoint = new ExtendedSequenceNumber("123", 0L); + KinesisClientLeaseBuilder builder = new KinesisClientLeaseBuilder().withCheckpoint(checkpoint) + .withConcurrencyToken(UUID.randomUUID()).withLastCounterIncrementNanos(0L).withLeaseCounter(0L) + .withOwnerSwitchesSinceCheckpoint(0L).withLeaseOwner("Self"); + + final List leases = new ArrayList<>(); + final List currentAssignments = new ArrayList<>(); + KinesisClientLease lease = builder.withLeaseKey(String.format("shardId-%03d", 1)).build(); + leases.add(lease); + currentAssignments.add(new ShardInfo(lease.getLeaseKey(), lease.getConcurrencyToken().toString(), + lease.getParentShardIds(), lease.getCheckpoint())); + + when(leaseCoordinator.getAssignments()).thenAnswer(new Answer>() { + @Override + public List answer(InvocationOnMock invocation) throws Throwable { + return leases; + } + }); + when(leaseCoordinator.getCurrentAssignments()).thenAnswer(new Answer>() { + @Override + public List answer(InvocationOnMock invocation) throws Throwable { + return currentAssignments; + } + }); + + IRecordProcessor processor = mock(IRecordProcessor.class); + when(recordProcessorFactory.createProcessor()).thenReturn(processor); + + GracefulShutdownCoordinator coordinator = mock(GracefulShutdownCoordinator.class); + when(coordinator.createGracefulShutdownCallable(any(Callable.class))).thenReturn(() -> true); + + Future gracefulShutdownFuture = mock(Future.class); + + when(coordinator.startGracefulShutdown(any(Callable.class))).thenReturn(gracefulShutdownFuture); + + Worker worker = new InjectableWorker("testRequestShutdown", + recordProcessorFactory, + config, + streamConfig, + INITIAL_POSITION_TRIM_HORIZON, + parentShardPollIntervalMillis, + shardSyncIntervalMillis, + cleanupLeasesUponShardCompletion, + leaseCoordinator, + leaseCoordinator, + executorService, + metricsFactory, + taskBackoffTimeMillis, + failoverTimeMillis, + false, + shardPrioritization) { + @Override + void postConstruct() { + this.gracefulShutdownCoordinator = coordinator; + } + }; + + when(executorService.submit(Matchers.> any())) + .thenAnswer(new ShutdownHandlingAnswer(taskFuture)); + when(taskFuture.isDone()).thenReturn(true); + when(taskFuture.get()).thenReturn(taskResult); + + worker.runProcessLoop(); + + verify(executorService, atLeastOnce()).submit(argThat(both(isA(MetricsCollectingTaskDecorator.class)) + .and(TaskTypeMatcher.isOfType(TaskType.BLOCK_ON_PARENT_SHARDS)))); + + worker.runProcessLoop(); + + verify(executorService, atLeastOnce()).submit(argThat( + both(isA(MetricsCollectingTaskDecorator.class)).and(TaskTypeMatcher.isOfType(TaskType.INITIALIZE)))); + + Future firstFuture = worker.startGracefulShutdown(); + Future secondFuture = worker.startGracefulShutdown(); + + assertThat(firstFuture, equalTo(secondFuture)); + verify(coordinator).startGracefulShutdown(any(Callable.class)); + + } + @Test public void testRequestShutdownNoLeases() throws Exception { @@ -810,10 +1047,22 @@ public class WorkerTest { when(recordProcessorFactory.createProcessor()).thenReturn(processor); - Worker worker = new Worker("testRequestShutdown", recordProcessorFactory, streamConfig, - INITIAL_POSITION_TRIM_HORIZON, parentShardPollIntervalMillis, shardSyncIntervalMillis, - cleanupLeasesUponShardCompletion, leaseCoordinator, leaseCoordinator, executorService, metricsFactory, - taskBackoffTimeMillis, failoverTimeMillis, false, shardPrioritization); + Worker worker = new Worker("testRequestShutdown", + recordProcessorFactory, + config, + streamConfig, + INITIAL_POSITION_TRIM_HORIZON, + parentShardPollIntervalMillis, + shardSyncIntervalMillis, + cleanupLeasesUponShardCompletion, + leaseCoordinator, + leaseCoordinator, + executorService, + metricsFactory, + taskBackoffTimeMillis, + failoverTimeMillis, + false, + shardPrioritization); when(executorService.submit(Matchers.> any())) .thenAnswer(new ShutdownHandlingAnswer(taskFuture)); @@ -830,7 +1079,7 @@ public class WorkerTest { verify(executorService, never()).submit(argThat( both(isA(MetricsCollectingTaskDecorator.class)).and(TaskTypeMatcher.isOfType(TaskType.INITIALIZE)))); - worker.requestShutdown(); + worker.createWorkerShutdownCallable().call(); worker.runProcessLoop(); verify(executorService, never()).submit(argThat(both(isA(MetricsCollectingTaskDecorator.class)) @@ -880,10 +1129,22 @@ public class WorkerTest { IRecordProcessor processor = mock(IRecordProcessor.class); when(recordProcessorFactory.createProcessor()).thenReturn(processor); - Worker worker = new Worker("testRequestShutdown", recordProcessorFactory, streamConfig, - INITIAL_POSITION_TRIM_HORIZON, parentShardPollIntervalMillis, shardSyncIntervalMillis, - cleanupLeasesUponShardCompletion, leaseCoordinator, leaseCoordinator, executorService, metricsFactory, - taskBackoffTimeMillis, failoverTimeMillis, false, shardPrioritization); + Worker worker = new Worker("testRequestShutdown", + recordProcessorFactory, + config, + streamConfig, + INITIAL_POSITION_TRIM_HORIZON, + parentShardPollIntervalMillis, + shardSyncIntervalMillis, + cleanupLeasesUponShardCompletion, + leaseCoordinator, + leaseCoordinator, + executorService, + metricsFactory, + taskBackoffTimeMillis, + failoverTimeMillis, + false, + shardPrioritization); when(executorService.submit(Matchers.> any())) .thenAnswer(new ShutdownHandlingAnswer(taskFuture)); @@ -909,7 +1170,7 @@ public class WorkerTest { .withField(InitializeTask.class, "shardInfo", equalTo(shardInfo2))))); worker.getShardInfoShardConsumerMap().remove(shardInfo2); - worker.requestShutdown(); + worker.createWorkerShutdownCallable().call(); leases.remove(1); currentAssignments.remove(1); worker.runProcessLoop(); @@ -981,10 +1242,22 @@ public class WorkerTest { IRecordProcessor processor = mock(IRecordProcessor.class); when(recordProcessorFactory.createProcessor()).thenReturn(processor); - Worker worker = new Worker("testRequestShutdown", recordProcessorFactory, streamConfig, - INITIAL_POSITION_TRIM_HORIZON, parentShardPollIntervalMillis, shardSyncIntervalMillis, - cleanupLeasesUponShardCompletion, leaseCoordinator, leaseCoordinator, executorService, metricsFactory, - taskBackoffTimeMillis, failoverTimeMillis, false, shardPrioritization); + Worker worker = new Worker("testRequestShutdown", + recordProcessorFactory, + config, + streamConfig, + INITIAL_POSITION_TRIM_HORIZON, + parentShardPollIntervalMillis, + shardSyncIntervalMillis, + cleanupLeasesUponShardCompletion, + leaseCoordinator, + leaseCoordinator, + executorService, + metricsFactory, + taskBackoffTimeMillis, + failoverTimeMillis, + false, + shardPrioritization); when(executorService.submit(Matchers.> any())) .thenAnswer(new ShutdownHandlingAnswer(taskFuture)); @@ -1086,10 +1359,22 @@ public class WorkerTest { IRecordProcessor processor = mock(IRecordProcessor.class); when(recordProcessorFactory.createProcessor()).thenReturn(processor); - Worker worker = new Worker("testRequestShutdown", recordProcessorFactory, streamConfig, - INITIAL_POSITION_TRIM_HORIZON, parentShardPollIntervalMillis, shardSyncIntervalMillis, - cleanupLeasesUponShardCompletion, leaseCoordinator, leaseCoordinator, executorService, metricsFactory, - taskBackoffTimeMillis, failoverTimeMillis, false, shardPrioritization); + Worker worker = new Worker("testRequestShutdown", + recordProcessorFactory, + config, + streamConfig, + INITIAL_POSITION_TRIM_HORIZON, + parentShardPollIntervalMillis, + shardSyncIntervalMillis, + cleanupLeasesUponShardCompletion, + leaseCoordinator, + leaseCoordinator, + executorService, + metricsFactory, + taskBackoffTimeMillis, + failoverTimeMillis, + false, + shardPrioritization); when(executorService.submit(Matchers.> any())) .thenAnswer(new ShutdownHandlingAnswer(taskFuture)); @@ -1158,10 +1443,22 @@ public class WorkerTest { IRecordProcessor processor = mock(IRecordProcessor.class); when(recordProcessorFactory.createProcessor()).thenReturn(processor); - Worker worker = new Worker("testRequestShutdown", recordProcessorFactory, streamConfig, - INITIAL_POSITION_TRIM_HORIZON, parentShardPollIntervalMillis, shardSyncIntervalMillis, - cleanupLeasesUponShardCompletion, leaseCoordinator, leaseCoordinator, executorService, metricsFactory, - taskBackoffTimeMillis, failoverTimeMillis, false, shardPrioritization); + Worker worker = new Worker("testRequestShutdown", + recordProcessorFactory, + config, + streamConfig, + INITIAL_POSITION_TRIM_HORIZON, + parentShardPollIntervalMillis, + shardSyncIntervalMillis, + cleanupLeasesUponShardCompletion, + leaseCoordinator, + leaseCoordinator, + executorService, + metricsFactory, + taskBackoffTimeMillis, + failoverTimeMillis, + false, + shardPrioritization); when(executorService.submit(Matchers.> any())) .thenAnswer(new ShutdownHandlingAnswer(taskFuture)); @@ -1194,6 +1491,306 @@ public class WorkerTest { } + @Test + public void testBuilderWithDefaultKinesisProxy() { + IRecordProcessorFactory recordProcessorFactory = mock(IRecordProcessorFactory.class); + Worker worker = new Worker.Builder() + .recordProcessorFactory(recordProcessorFactory) + .config(config) + .build(); + Assert.assertNotNull(worker.getStreamConfig().getStreamProxy()); + Assert.assertTrue(worker.getStreamConfig().getStreamProxy() instanceof KinesisProxy); + } + + @Test + public void testBuilderWhenKinesisProxyIsSet() { + IRecordProcessorFactory recordProcessorFactory = mock(IRecordProcessorFactory.class); + // Create an instance of KinesisLocalFileProxy for injection and validation + IKinesisProxy kinesisProxy = mock(KinesisLocalFileProxy.class); + Worker worker = new Worker.Builder() + .recordProcessorFactory(recordProcessorFactory) + .config(config) + .kinesisProxy(kinesisProxy) + .build(); + Assert.assertNotNull(worker.getStreamConfig().getStreamProxy()); + Assert.assertTrue(worker.getStreamConfig().getStreamProxy() instanceof KinesisLocalFileProxy); + } + + @Test + public void testBuilderForWorkerStateListener() { + IRecordProcessorFactory recordProcessorFactory = mock(IRecordProcessorFactory.class); + Worker worker = new Worker.Builder() + .recordProcessorFactory(recordProcessorFactory) + .config(config) + .build(); + Assert.assertTrue(worker.getWorkerStateChangeListener() instanceof NoOpWorkerStateChangeListener); + } + + @Test + public void testBuilderWhenWorkerStateListenerIsSet() { + IRecordProcessorFactory recordProcessorFactory = mock(IRecordProcessorFactory.class); + Worker worker = new Worker.Builder() + .recordProcessorFactory(recordProcessorFactory) + .workerStateChangeListener(workerStateChangeListener) + .config(config) + .build(); + Assert.assertSame(workerStateChangeListener, worker.getWorkerStateChangeListener()); + } + + @Test + public void testWorkerStateListenerStatePassesThroughCreatedState() { + IRecordProcessorFactory recordProcessorFactory = mock(IRecordProcessorFactory.class); + new Worker.Builder() + .recordProcessorFactory(recordProcessorFactory) + .workerStateChangeListener(workerStateChangeListener) + .config(config) + .build(); + + verify(workerStateChangeListener, times(1)).onWorkerStateChange(eq(WorkerState.CREATED)); + } + + @Test + public void testWorkerStateChangeListenerGoesThroughStates() throws Exception { + + final CountDownLatch workerInitialized = new CountDownLatch(1); + final CountDownLatch workerStarted = new CountDownLatch(1); + final IRecordProcessorFactory recordProcessorFactory = mock(IRecordProcessorFactory.class); + final IRecordProcessor processor = mock(IRecordProcessor.class); + + ExtendedSequenceNumber checkpoint = new ExtendedSequenceNumber("123", 0L); + KinesisClientLeaseBuilder builder = new KinesisClientLeaseBuilder().withCheckpoint(checkpoint) + .withConcurrencyToken(UUID.randomUUID()).withLastCounterIncrementNanos(0L).withLeaseCounter(0L) + .withOwnerSwitchesSinceCheckpoint(0L).withLeaseOwner("Self"); + final List leases = new ArrayList<>(); + KinesisClientLease lease = builder.withLeaseKey(String.format("shardId-%03d", 1)).build(); + leases.add(lease); + + doAnswer(new Answer() { + @Override + public Boolean answer(InvocationOnMock invocation) throws Throwable { + workerInitialized.countDown(); + return true; + } + }).when(leaseManager).waitUntilLeaseTableExists(anyLong(), anyLong()); + doAnswer(new Answer() { + @Override + public IRecordProcessor answer(InvocationOnMock invocation) throws Throwable { + workerStarted.countDown(); + return processor; + } + }).when(recordProcessorFactory).createProcessor(); + + when(config.getWorkerIdentifier()).thenReturn("Self"); + when(leaseManager.listLeases()).thenReturn(leases); + when(leaseManager.renewLease(leases.get(0))).thenReturn(true); + when(executorService.submit(Matchers.> any())) + .thenAnswer(new ShutdownHandlingAnswer(taskFuture)); + when(taskFuture.isDone()).thenReturn(true); + when(taskFuture.get()).thenReturn(taskResult); + when(taskResult.isShardEndReached()).thenReturn(true); + + Worker worker = new Worker.Builder() + .recordProcessorFactory(recordProcessorFactory) + .config(config) + .leaseManager(leaseManager) + .kinesisProxy(kinesisProxy) + .execService(executorService) + .workerStateChangeListener(workerStateChangeListener) + .build(); + + verify(workerStateChangeListener, times(1)).onWorkerStateChange(eq(WorkerState.CREATED)); + + WorkerThread workerThread = new WorkerThread(worker); + workerThread.start(); + + workerInitialized.await(); + verify(workerStateChangeListener, times(1)).onWorkerStateChange(eq(WorkerState.INITIALIZING)); + + workerStarted.await(); + verify(workerStateChangeListener, times(1)).onWorkerStateChange(eq(WorkerState.STARTED)); + + boolean workerShutdown = worker.createGracefulShutdownCallable() + .call(); + + verify(workerStateChangeListener, times(1)).onWorkerStateChange(eq(WorkerState.SHUT_DOWN)); + } + + @Test + public void testBuilderWithDefaultLeaseManager() { + IRecordProcessorFactory recordProcessorFactory = mock(IRecordProcessorFactory.class); + + Worker worker = new Worker.Builder() + .recordProcessorFactory(recordProcessorFactory) + .config(config) + .build(); + + Assert.assertNotNull(worker.getLeaseCoordinator().getLeaseManager()); + } + + @SuppressWarnings("unchecked") + @Test + public void testBuilderWhenLeaseManagerIsSet() { + IRecordProcessorFactory recordProcessorFactory = mock(IRecordProcessorFactory.class); + // Create an instance of ILeaseManager for injection and validation + ILeaseManager leaseManager = (ILeaseManager) mock(ILeaseManager.class); + Worker worker = new Worker.Builder() + .recordProcessorFactory(recordProcessorFactory) + .config(config) + .leaseManager(leaseManager) + .build(); + + Assert.assertSame(leaseManager, worker.getLeaseCoordinator().getLeaseManager()); + } + + @Test + public void testBuilderSetRegionAndEndpointToClient() { + IRecordProcessorFactory recordProcessorFactory = mock(IRecordProcessorFactory.class); + final String endpoint = "TestEndpoint"; + KinesisClientLibConfiguration config = new KinesisClientLibConfiguration("TestApp", null, null, null) + .withRegionName(Regions.US_WEST_2.getName()) + .withKinesisEndpoint(endpoint) + .withDynamoDBEndpoint(endpoint); + + AmazonKinesis kinesisClient = spy(AmazonKinesisClientBuilder.standard().withRegion(Regions.US_WEST_2).build()); + AmazonDynamoDB dynamoDBClient = spy(AmazonDynamoDBClientBuilder.standard().withRegion(Regions.US_WEST_2).build()); + AmazonCloudWatch cloudWatchClient = spy(AmazonCloudWatchClientBuilder.standard().withRegion(Regions.US_WEST_2).build()); + + new Worker.Builder().recordProcessorFactory(recordProcessorFactory).config(config) + .kinesisClient(kinesisClient) + .dynamoDBClient(dynamoDBClient) + .cloudWatchClient(cloudWatchClient) + .build(); + + verify(kinesisClient, times(1)).setRegion(eq(RegionUtils.getRegion(config.getRegionName()))); + verify(dynamoDBClient, times(1)).setRegion(eq(RegionUtils.getRegion(config.getRegionName()))); + verify(cloudWatchClient, times(2)).setRegion(eq(RegionUtils.getRegion(config.getRegionName()))); + + verify(kinesisClient, times(1)).setEndpoint(eq(endpoint)); + verify(dynamoDBClient, times(1)).setEndpoint(eq(endpoint)); + verify(cloudWatchClient, never()).setEndpoint(anyString()); + } + + @Test + public void testBuilderSetRegionToClient() { + IRecordProcessorFactory recordProcessorFactory = mock(IRecordProcessorFactory.class); + String region = Regions.US_WEST_2.getName(); + KinesisClientLibConfiguration config = new KinesisClientLibConfiguration("TestApp", null, null, null) + .withRegionName(region); + + Worker.Builder builder = new Worker.Builder(); + + AmazonKinesis kinesisClient = spy(AmazonKinesisClientBuilder.standard().withRegion(Regions.US_WEST_2).build()); + AmazonDynamoDB dynamoDBClient = spy(AmazonDynamoDBClientBuilder.standard().withRegion(Regions.US_WEST_2).build()); + AmazonCloudWatch cloudWatchClient = spy(AmazonCloudWatchClientBuilder.standard().withRegion(Regions.US_WEST_2).build()); + + builder.recordProcessorFactory(recordProcessorFactory).config(config) + .kinesisClient(kinesisClient) + .dynamoDBClient(dynamoDBClient) + .cloudWatchClient(cloudWatchClient) + .build(); + + verify(kinesisClient, times(1)).setRegion(eq(RegionUtils.getRegion(config.getRegionName()))); + verify(dynamoDBClient, times(1)).setRegion(eq(RegionUtils.getRegion(config.getRegionName()))); + verify(cloudWatchClient, times(2)).setRegion(eq(RegionUtils.getRegion(config.getRegionName()))); + + verify(kinesisClient, never()).setEndpoint(any()); + verify(dynamoDBClient, never()).setEndpoint(any()); + verify(cloudWatchClient, never()).setEndpoint(any()); + } + + @Test + public void testBuilderGenerateClients() { + IRecordProcessorFactory recordProcessorFactory = mock(IRecordProcessorFactory.class); + KinesisClientLibConfiguration config = new KinesisClientLibConfiguration("TestApp", null, null, null); + Worker.Builder builder = spy(new Worker.Builder().recordProcessorFactory(recordProcessorFactory).config(config)); + ArgumentCaptor builderCaptor = ArgumentCaptor.forClass(AwsClientBuilder.class); + + assertNull(builder.getKinesisClient()); + assertNull(builder.getDynamoDBClient()); + assertNull(builder.getCloudWatchClient()); + + builder.build(); + + assertTrue(builder.getKinesisClient() instanceof AmazonKinesis); + assertTrue(builder.getDynamoDBClient() instanceof AmazonDynamoDB); + assertTrue(builder.getCloudWatchClient() instanceof AmazonCloudWatch); + + verify(builder, times(3)).createClient( + builderCaptor.capture(), eq(null), any(ClientConfiguration.class), eq(null), eq(null)); + + builderCaptor.getAllValues().forEach(clientBuilder -> { + assertTrue(clientBuilder.getRegion().equals(Regions.US_EAST_1.getName())); + }); + } + + @Test + public void testBuilderGenerateClientsWithRegion() { + IRecordProcessorFactory recordProcessorFactory = mock(IRecordProcessorFactory.class); + String region = Regions.US_WEST_2.getName(); + KinesisClientLibConfiguration config = new KinesisClientLibConfiguration("TestApp", null, null, null) + .withRegionName(region); + ArgumentCaptor builderCaptor = ArgumentCaptor.forClass(AwsClientBuilder.class); + + Worker.Builder builder = spy(new Worker.Builder()); + + builder.recordProcessorFactory(recordProcessorFactory).config(config).build(); + + verify(builder, times(3)).createClient( + builderCaptor.capture(), eq(null), any(ClientConfiguration.class), eq(null), eq(region)); + builderCaptor.getAllValues().forEach(clientBuilder -> { + assertTrue(clientBuilder.getRegion().equals(region)); + }); + } + + @Test + public void testBuilderGenerateClientsWithEndpoint() { + IRecordProcessorFactory recordProcessorFactory = mock(IRecordProcessorFactory.class); + String region = Regions.US_WEST_2.getName(); + String endpointUrl = "TestEndpoint"; + KinesisClientLibConfiguration config = new KinesisClientLibConfiguration("TestApp", null, null, null) + .withRegionName(region).withKinesisEndpoint(endpointUrl).withDynamoDBEndpoint(endpointUrl); + + Worker.Builder builder = spy(new Worker.Builder()); + + builder.recordProcessorFactory(recordProcessorFactory).config(config).build(); + + verify(builder, times(2)).createClient( + any(AwsClientBuilder.class), eq(null), any(ClientConfiguration.class), eq(endpointUrl), eq(region)); + verify(builder, times(1)).createClient( + any(AwsClientBuilder.class), eq(null), any(ClientConfiguration.class), eq(null), eq(region)); + } + + private abstract class InjectableWorker extends Worker { + InjectableWorker(String applicationName, IRecordProcessorFactory recordProcessorFactory, + KinesisClientLibConfiguration config, StreamConfig streamConfig, + InitialPositionInStreamExtended initialPositionInStream, + long parentShardPollIntervalMillis, long shardSyncIdleTimeMillis, + boolean cleanupLeasesUponShardCompletion, ICheckpoint checkpoint, + KinesisClientLibLeaseCoordinator leaseCoordinator, ExecutorService execService, + IMetricsFactory metricsFactory, long taskBackoffTimeMillis, long failoverTimeMillis, + boolean skipShardSyncAtWorkerInitializationIfLeasesExist, ShardPrioritization shardPrioritization) { + super(applicationName, + recordProcessorFactory, + config, + streamConfig, + initialPositionInStream, + parentShardPollIntervalMillis, + shardSyncIdleTimeMillis, + cleanupLeasesUponShardCompletion, + checkpoint, + leaseCoordinator, + execService, + metricsFactory, + taskBackoffTimeMillis, + failoverTimeMillis, + skipShardSyncAtWorkerInitializationIfLeasesExist, + shardPrioritization); + postConstruct(); + } + + abstract void postConstruct(); + } + private KinesisClientLease makeLease(ExtendedSequenceNumber checkpoint, int shardId) { return new KinesisClientLeaseBuilder().withCheckpoint(checkpoint).withConcurrencyToken(UUID.randomUUID()) .withLastCounterIncrementNanos(0L).withLeaseCounter(0L).withOwnerSwitchesSinceCheckpoint(0L) @@ -1420,14 +2017,15 @@ public class WorkerTest { lease.setCheckpoint(ExtendedSequenceNumber.AT_TIMESTAMP); initialLeases.add(lease); } - runAndTestWorker(shardList, threadPoolSize, initialLeases, callProcessRecordsForEmptyRecordList, numberOfRecordsPerShard); + runAndTestWorker(shardList, threadPoolSize, initialLeases, callProcessRecordsForEmptyRecordList, numberOfRecordsPerShard, config); } private void runAndTestWorker(List shardList, - int threadPoolSize, - List initialLeases, - boolean callProcessRecordsForEmptyRecordList, - int numberOfRecordsPerShard) throws Exception { + int threadPoolSize, + List initialLeases, + boolean callProcessRecordsForEmptyRecordList, + int numberOfRecordsPerShard, + KinesisClientLibConfiguration clientConfig) throws Exception { File file = KinesisLocalFileDataCreator.generateTempDataFile(shardList, numberOfRecordsPerShard, "unitTestWT001"); IKinesisProxy fileBasedProxy = new KinesisLocalFileProxy(file.getAbsolutePath()); @@ -1439,7 +2037,7 @@ public class WorkerTest { WorkerThread workerThread = runWorker( shardList, initialLeases, callProcessRecordsForEmptyRecordList, failoverTimeMillis, - numberOfRecordsPerShard, fileBasedProxy, recordProcessorFactory, executorService, nullMetricsFactory); + numberOfRecordsPerShard, fileBasedProxy, recordProcessorFactory, executorService, nullMetricsFactory, clientConfig); // TestStreamlet will release the semaphore once for every record it processes recordCounter.acquire(numberOfRecordsPerShard * shardList.size()); @@ -1456,14 +2054,15 @@ public class WorkerTest { } private WorkerThread runWorker(List shardList, - List initialLeases, - boolean callProcessRecordsForEmptyRecordList, - long failoverTimeMillis, - int numberOfRecordsPerShard, - IKinesisProxy kinesisProxy, - IRecordProcessorFactory recordProcessorFactory, - ExecutorService executorService, - IMetricsFactory metricsFactory) throws Exception { + List initialLeases, + boolean callProcessRecordsForEmptyRecordList, + long failoverTimeMillis, + int numberOfRecordsPerShard, + IKinesisProxy kinesisProxy, + IRecordProcessorFactory recordProcessorFactory, + ExecutorService executorService, + IMetricsFactory metricsFactory, + KinesisClientLibConfiguration clientConfig) throws Exception { final String stageName = "testStageName"; final int maxRecords = 2; @@ -1495,6 +2094,7 @@ public class WorkerTest { Worker worker = new Worker(stageName, recordProcessorFactory, + clientConfig, streamConfig, INITIAL_POSITION_TRIM_HORIZON, parentShardPollIntervalMillis, shardSyncIntervalMillis, @@ -1507,7 +2107,7 @@ public class WorkerTest { failoverTimeMillis, KinesisClientLibConfiguration.DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST, shardPrioritization); - + WorkerThread workerThread = new WorkerThread(worker); workerThread.start(); return workerThread; diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisLocalFileProxyFactory.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisLocalFileProxyFactory.java index 8a053ec4..f14ec63b 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisLocalFileProxyFactory.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisLocalFileProxyFactory.java @@ -33,7 +33,6 @@ public class KinesisLocalFileProxyFactory implements IKinesisProxyFactory { private static final String DEFAULT_TEST_PROXY_FILE = "defaultKinesisProxyLocalFile"; private IKinesisProxy testKinesisProxy; - /** * @param fileName File to be used for stream data. @@ -60,5 +59,4 @@ public class KinesisLocalFileProxyFactory implements IKinesisProxyFactory { public IKinesisProxy getProxy(String streamARN) { return testKinesisProxy; } - } diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxyTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxyTest.java index 2c1107b2..845d2208 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxyTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxyTest.java @@ -1,23 +1,55 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ package com.amazonaws.services.kinesis.clientlibrary.proxies; +import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.Matchers.both; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasProperty; import static org.hamcrest.Matchers.isA; +import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; import static org.mockito.Matchers.any; import static org.mockito.Matchers.argThat; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; +import java.util.stream.Collectors; -import com.amazonaws.AmazonServiceException; +import org.apache.commons.lang3.StringUtils; +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -25,53 +57,101 @@ import org.mockito.ArgumentMatcher; import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; +import com.amazonaws.AmazonServiceException; import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.services.kinesis.AmazonKinesisClient; +import com.amazonaws.services.dynamodbv2.streamsadapter.AmazonDynamoDBStreamsAdapterClient; +import com.amazonaws.services.dynamodbv2.streamsadapter.AmazonDynamoDBStreamsAdapterClientChild; +import com.amazonaws.services.kinesis.AmazonKinesis; +import com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisClientLibConfiguration; import com.amazonaws.services.kinesis.model.DescribeStreamRequest; import com.amazonaws.services.kinesis.model.DescribeStreamResult; import com.amazonaws.services.kinesis.model.GetShardIteratorRequest; import com.amazonaws.services.kinesis.model.GetShardIteratorResult; import com.amazonaws.services.kinesis.model.LimitExceededException; +import com.amazonaws.services.kinesis.model.ListShardsRequest; +import com.amazonaws.services.kinesis.model.ListShardsResult; +import com.amazonaws.services.kinesis.model.ResourceInUseException; import com.amazonaws.services.kinesis.model.Shard; import com.amazonaws.services.kinesis.model.ShardIteratorType; import com.amazonaws.services.kinesis.model.StreamDescription; import com.amazonaws.services.kinesis.model.StreamStatus; -import junit.framework.Assert; +import lombok.AllArgsConstructor; @RunWith(MockitoJUnitRunner.class) public class KinesisProxyTest { private static final String TEST_STRING = "TestString"; - private static final long BACKOFF_TIME = 10L; - private static final int RETRY_TIMES = 50; + private static final long DESCRIBE_STREAM_BACKOFF_TIME = 10L; + private static final long LIST_SHARDS_BACKOFF_TIME = 10L; + private static final int DESCRIBE_STREAM_RETRY_TIMES = 3; + private static final int LIST_SHARDS_RETRY_TIMES = 3; + private static final String NEXT_TOKEN = "NextToken"; + private static final String SHARD_1 = "shard-1"; + private static final String SHARD_2 = "shard-2"; + private static final String SHARD_3 = "shard-3"; + private static final String SHARD_4 = "shard-4"; + private static final String NOT_CACHED_SHARD = "ShardId-0005"; + private static final String NEVER_PRESENT_SHARD = "ShardId-0010"; @Mock - private AmazonKinesisClient mockClient; + private AmazonKinesis mockClient; + @Mock + private AmazonDynamoDBStreamsAdapterClient mockDDBStreamClient; + @Mock + private AmazonDynamoDBStreamsAdapterClientChild mockDDBChildClient; @Mock private AWSCredentialsProvider mockCredentialsProvider; @Mock private GetShardIteratorResult shardIteratorResult; + @Mock + private DescribeStreamResult describeStreamResult; + @Mock + private StreamDescription streamDescription; + @Mock + private Shard shard; + @Mock + private KinesisClientLibConfiguration config; + @Mock + private ListShardsResult listShardsResult; + private KinesisProxy proxy; + private KinesisProxy ddbProxy; + private KinesisProxy ddbChildProxy; // Test shards for verifying. private Set shardIdSet; private List shards; + private Map shardMap; + + private List updatedShards; + private Map updatedShardMap; @Before public void setUpTest() { - // Set up kinesis proxy - proxy = new KinesisProxy(TEST_STRING, mockCredentialsProvider, mockClient, BACKOFF_TIME, RETRY_TIMES); - when(mockCredentialsProvider.getCredentials()).thenReturn(null); + // Set up kinesis ddbProxy + when(config.getStreamName()).thenReturn(TEST_STRING); + when(config.getListShardsBackoffTimeInMillis()).thenReturn(LIST_SHARDS_BACKOFF_TIME); + when(config.getMaxListShardsRetryAttempts()).thenReturn(LIST_SHARDS_RETRY_TIMES); + when(config.getKinesisCredentialsProvider()).thenReturn(mockCredentialsProvider); + + proxy = new KinesisProxy(config, mockClient); + ddbProxy = new KinesisProxy(TEST_STRING, mockCredentialsProvider, mockDDBStreamClient, + DESCRIBE_STREAM_BACKOFF_TIME, DESCRIBE_STREAM_RETRY_TIMES, LIST_SHARDS_BACKOFF_TIME, + LIST_SHARDS_RETRY_TIMES); + ddbChildProxy = new KinesisProxy(TEST_STRING, mockCredentialsProvider, mockDDBChildClient, + DESCRIBE_STREAM_BACKOFF_TIME, DESCRIBE_STREAM_RETRY_TIMES, LIST_SHARDS_BACKOFF_TIME, + LIST_SHARDS_RETRY_TIMES); + // Set up test shards - shardIdSet = new HashSet<>(); - shards = new ArrayList<>(); - String[] shardIds = new String[] { "shard-1", "shard-2", "shard-3", "shard-4" }; - for (String shardId : shardIds) { - Shard shard = new Shard(); - shard.setShardId(shardId); - shards.add(shard); - shardIdSet.add(shardId); - } + List shardIds = Arrays.asList(SHARD_1, SHARD_2, SHARD_3, SHARD_4); + shardIdSet = new HashSet<>(shardIds); + shards = shardIds.stream().map(shardId -> new Shard().withShardId(shardId)).collect(Collectors.toList()); + shardMap = shards.stream().collect(Collectors.toMap(Shard::getShardId, Function.identity())); + + updatedShards = new ArrayList<>(shards); + updatedShards.add(new Shard().withShardId(NOT_CACHED_SHARD)); + updatedShardMap = updatedShards.stream().collect(Collectors.toMap(Shard::getShardId, Function.identity())); + } @Test @@ -81,12 +161,12 @@ public class KinesisProxyTest { // Second call describeStream returning response with rest shards. DescribeStreamResult responseWithMoreData = createGetStreamInfoResponse(shards.subList(0, 2), true); DescribeStreamResult responseFinal = createGetStreamInfoResponse(shards.subList(2, shards.size()), false); - doReturn(responseWithMoreData).when(mockClient).describeStream(argThat(new IsRequestWithStartShardId(null))); - doReturn(responseFinal).when(mockClient) - .describeStream(argThat(new IsRequestWithStartShardId(shards.get(1).getShardId()))); + doReturn(responseWithMoreData).when(mockDDBStreamClient).describeStream(argThat(new IsRequestWithStartShardId(null))); + doReturn(responseFinal).when(mockDDBStreamClient) + .describeStream(argThat(new OldIsRequestWithStartShardId(shards.get(1).getShardId()))); - Set resultShardIdSets = proxy.getAllShardIds(); - Assert.assertTrue("Result set should equal to Test set", shardIdSet.equals(resultShardIdSets)); + Set resultShardIdSets = ddbProxy.getAllShardIds(); + assertThat("Result set should equal to Test set", shardIdSet, equalTo(resultShardIdSets)); } @Test @@ -95,43 +175,321 @@ public class KinesisProxyTest { // First call describeStream throwing LimitExceededException; // Second call describeStream returning shards list. DescribeStreamResult response = createGetStreamInfoResponse(shards, false); - doThrow(new LimitExceededException("Test Exception")).doReturn(response).when(mockClient) - .describeStream(argThat(new IsRequestWithStartShardId(null))); + doThrow(new LimitExceededException("Test Exception")).doReturn(response).when(mockDDBStreamClient) + .describeStream(argThat(new OldIsRequestWithStartShardId(null))); - Set resultShardIdSet = proxy.getAllShardIds(); - Assert.assertTrue("Result set should equal to Test set", shardIdSet.equals(resultShardIdSet)); + Set resultShardIdSet = ddbProxy.getAllShardIds(); + assertThat("Result set should equal to Test set", shardIdSet, equalTo(resultShardIdSet)); } @Test public void testValidShardIteratorType() { - when(mockClient.getShardIterator(any(GetShardIteratorRequest.class))).thenReturn(shardIteratorResult); + when(mockDDBStreamClient.getShardIterator(any(GetShardIteratorRequest.class))).thenReturn(shardIteratorResult); String expectedShardIteratorType = ShardIteratorType.AFTER_SEQUENCE_NUMBER.toString(); - proxy.getIterator("Shard-001", expectedShardIteratorType, "1234"); + ddbProxy.getIterator("Shard-001", expectedShardIteratorType, "1234"); - verify(mockClient).getShardIterator(argThat(both(isA(GetShardIteratorRequest.class)) + verify(mockDDBStreamClient).getShardIterator(argThat(both(isA(GetShardIteratorRequest.class)) .and(hasProperty("shardIteratorType", equalTo(expectedShardIteratorType))))); } @Test public void testInvalidShardIteratorIsntChanged() { - when(mockClient.getShardIterator(any(GetShardIteratorRequest.class))).thenReturn(shardIteratorResult); + when(mockDDBStreamClient.getShardIterator(any(GetShardIteratorRequest.class))).thenReturn(shardIteratorResult); String expectedShardIteratorType = ShardIteratorType.AT_TIMESTAMP.toString(); - proxy.getIterator("Shard-001", expectedShardIteratorType, "1234"); + ddbProxy.getIterator("Shard-001", expectedShardIteratorType, "1234"); - verify(mockClient).getShardIterator(argThat(both(isA(GetShardIteratorRequest.class)) + verify(mockDDBStreamClient).getShardIterator(argThat(both(isA(GetShardIteratorRequest.class)) .and(hasProperty("shardIteratorType", equalTo(expectedShardIteratorType))))); } @Test(expected = AmazonServiceException.class) - public void testNullShardIteratorType() { - when(mockClient.getShardIterator(any(GetShardIteratorRequest.class))).thenThrow(new AmazonServiceException("expected null")); + public void testNullShardIteratorType() throws Exception { + when(mockDDBStreamClient.getShardIterator(any(GetShardIteratorRequest.class))).thenThrow(new AmazonServiceException("expected null")); String expectedShardIteratorType = null; - proxy.getIterator("Shard-001", expectedShardIteratorType, "1234"); + ddbProxy.getIterator("Shard-001", expectedShardIteratorType, "1234"); - verify(mockClient).getShardIterator(argThat(both(isA(GetShardIteratorRequest.class)) + verify(mockDDBStreamClient).getShardIterator(argThat(both(isA(GetShardIteratorRequest.class)) .and(hasProperty("shardIteratorType", nullValue(String.class))))); } + @Test(expected = AmazonServiceException.class) + public void testGetStreamInfoFails() { + when(mockDDBStreamClient.describeStream(any(DescribeStreamRequest.class))).thenThrow(new AmazonServiceException("Test")); + try { + ddbProxy.getShardList(); + } finally { + verify(mockDDBStreamClient).describeStream(any(DescribeStreamRequest.class)); + } + } + + @Test + public void testGetStreamInfoThrottledOnce() throws Exception { + when(mockDDBStreamClient.describeStream(any(DescribeStreamRequest.class))).thenThrow(new LimitExceededException("Test")) + .thenReturn(describeStreamResult); + when(describeStreamResult.getStreamDescription()).thenReturn(streamDescription); + when(streamDescription.getHasMoreShards()).thenReturn(false); + when(streamDescription.getStreamStatus()).thenReturn(StreamStatus.ACTIVE.name()); + List expectedShards = Collections.singletonList(shard); + when(streamDescription.getShards()).thenReturn(expectedShards); + + List actualShards = ddbProxy.getShardList(); + + assertThat(actualShards, equalTo(expectedShards)); + + verify(mockDDBStreamClient, times(2)).describeStream(any(DescribeStreamRequest.class)); + verify(describeStreamResult, times(3)).getStreamDescription(); + verify(streamDescription).getStreamStatus(); + verify(streamDescription).isHasMoreShards(); + } + + @Test(expected = LimitExceededException.class) + public void testGetStreamInfoThrottledAll() throws Exception { + when(mockDDBStreamClient.describeStream(any(DescribeStreamRequest.class))).thenThrow(new LimitExceededException("Test")); + + ddbProxy.getShardList(); + } + + @Test + public void testGetStreamInfoStoresOffset() throws Exception { + when(describeStreamResult.getStreamDescription()).thenReturn(streamDescription); + when(streamDescription.getStreamStatus()).thenReturn(StreamStatus.ACTIVE.name()); + Shard shard1 = mock(Shard.class); + Shard shard2 = mock(Shard.class); + Shard shard3 = mock(Shard.class); + List shardList1 = Collections.singletonList(shard1); + List shardList2 = Collections.singletonList(shard2); + List shardList3 = Collections.singletonList(shard3); + + String shardId1 = "ShardId-0001"; + String shardId2 = "ShardId-0002"; + String shardId3 = "ShardId-0003"; + + when(shard1.getShardId()).thenReturn(shardId1); + when(shard2.getShardId()).thenReturn(shardId2); + when(shard3.getShardId()).thenReturn(shardId3); + + when(streamDescription.getShards()).thenReturn(shardList1).thenReturn(shardList2).thenReturn(shardList3); + when(streamDescription.isHasMoreShards()).thenReturn(true, true, false); + when(mockDDBStreamClient.describeStream(argThat(describeWithoutShardId()))).thenReturn(describeStreamResult); + + when(mockDDBStreamClient.describeStream(argThat(describeWithShardId(shardId1)))) + .thenThrow(new LimitExceededException("1"), new LimitExceededException("2"), + new LimitExceededException("3")) + .thenReturn(describeStreamResult); + + when(mockDDBStreamClient.describeStream(argThat(describeWithShardId(shardId2)))).thenReturn(describeStreamResult); + + boolean limitExceeded = false; + try { + ddbProxy.getShardList(); + } catch (LimitExceededException le) { + limitExceeded = true; + } + assertThat(limitExceeded, equalTo(true)); + List actualShards = ddbProxy.getShardList(); + List expectedShards = Arrays.asList(shard1, shard2, shard3); + + assertThat(actualShards, equalTo(expectedShards)); + + verify(mockDDBStreamClient).describeStream(argThat(describeWithoutShardId())); + verify(mockDDBStreamClient, times(4)).describeStream(argThat(describeWithShardId(shardId1))); + verify(mockDDBStreamClient).describeStream(argThat(describeWithShardId(shardId2))); + + } + + @Test + public void testListShardsWithMoreDataAvailable() { + ListShardsResult responseWithMoreData = new ListShardsResult().withShards(shards.subList(0, 2)).withNextToken(NEXT_TOKEN); + ListShardsResult responseFinal = new ListShardsResult().withShards(shards.subList(2, shards.size())).withNextToken(null); + when(mockClient.listShards(argThat(initialListShardsRequestMatcher()))).thenReturn(responseWithMoreData); + when(mockClient.listShards(argThat(listShardsNextToken(NEXT_TOKEN)))).thenReturn(responseFinal); + + Set resultShardIdSets = proxy.getAllShardIds(); + assertEquals(shardIdSet, resultShardIdSets); + } + + @Test + public void testListShardsWithLimiteExceededException() { + ListShardsResult result = new ListShardsResult().withShards(shards); + when(mockClient.listShards(argThat(initialListShardsRequestMatcher()))).thenThrow(LimitExceededException.class).thenReturn(result); + + Set resultShardIdSet = proxy.getAllShardIds(); + assertEquals(shardIdSet, resultShardIdSet); + } + + @Test(expected = AmazonServiceException.class) + public void testListShardsFails() { + when(mockClient.listShards(any(ListShardsRequest.class))).thenThrow(AmazonServiceException.class); + try { + proxy.getShardList(); + } finally { + verify(mockClient).listShards(any(ListShardsRequest.class)); + } + } + + @Test + public void testListShardsThrottledOnce() { + List expected = Collections.singletonList(shard); + ListShardsResult result = new ListShardsResult().withShards(expected); + when(mockClient.listShards(argThat(initialListShardsRequestMatcher()))).thenThrow(LimitExceededException.class) + .thenReturn(result); + + List actualShards = proxy.getShardList(); + + assertEquals(expected, actualShards); + verify(mockClient, times(2)).listShards(argThat(initialListShardsRequestMatcher())); + } + + @Test(expected = LimitExceededException.class) + public void testListShardsThrottledAll() { + when(mockClient.listShards(argThat(initialListShardsRequestMatcher()))).thenThrow(LimitExceededException.class); + proxy.getShardList(); + } + + @Test + public void testStreamNotInCorrectStatus() { + when(mockClient.listShards(argThat(initialListShardsRequestMatcher()))).thenThrow(ResourceInUseException.class); + assertNull(proxy.getShardList()); + } + + @Test + public void testGetShardListWithDDBChildClient() { + DescribeStreamResult responseWithMoreData = createGetStreamInfoResponse(shards.subList(0, 2), true); + DescribeStreamResult responseFinal = createGetStreamInfoResponse(shards.subList(2, shards.size()), false); + doReturn(responseWithMoreData).when(mockDDBChildClient).describeStream(argThat(new IsRequestWithStartShardId(null))); + doReturn(responseFinal).when(mockDDBChildClient) + .describeStream(argThat(new OldIsRequestWithStartShardId(shards.get(1).getShardId()))); + + Set resultShardIdSets = ddbChildProxy.getAllShardIds(); + assertThat("Result set should equal to Test set", shardIdSet, equalTo(resultShardIdSets)); + } + + @Test + public void testGetShardCacheEmpty() { + mockListShardsForSingleResponse(shards); + Shard shard = proxy.getShard(SHARD_1); + assertThat(shard.getShardId(), equalTo(SHARD_1)); + verify(mockClient).listShards(any()); + } + + @Test + public void testGetShardCacheNotLoadingWhenCacheHit() { + proxy.setCachedShardMap(shardMap); + Shard shard = proxy.getShard(SHARD_1); + + assertThat(shard, notNullValue()); + assertThat(shard.getShardId(), equalTo(SHARD_1)); + + verify(mockClient, never()).listShards(any()); + } + + @Test + public void testGetShardCacheLoadAfterMaxMisses() { + proxy.setCachedShardMap(shardMap); + proxy.setCacheMisses(new AtomicInteger(KinesisProxy.MAX_CACHE_MISSES_BEFORE_RELOAD)); + + mockListShardsForSingleResponse(updatedShards); + + Shard shard = proxy.getShard(NOT_CACHED_SHARD); + assertThat(shard, notNullValue()); + assertThat(shard.getShardId(), equalTo(NOT_CACHED_SHARD)); + + assertThat(proxy.getCacheMisses().get(), equalTo(0)); + + verify(mockClient).listShards(any()); + + } + + @Test + public void testGetShardCacheNonLoadBeforeMaxMisses() { + proxy.setCachedShardMap(shardMap); + proxy.setLastCacheUpdateTime(Instant.now()); + proxy.setCacheMisses(new AtomicInteger(KinesisProxy.MAX_CACHE_MISSES_BEFORE_RELOAD - 1)); + + Shard shard = proxy.getShard(NOT_CACHED_SHARD); + assertThat(shard, nullValue()); + assertThat(proxy.getCacheMisses().get(), equalTo(KinesisProxy.MAX_CACHE_MISSES_BEFORE_RELOAD)); + verify(mockClient, never()).listShards(any()); + } + + @Test + public void testGetShardCacheMissesResetsAfterLoad() { + proxy.setCachedShardMap(shardMap); + proxy.setLastCacheUpdateTime(Instant.now()); + proxy.setCacheMisses(new AtomicInteger(KinesisProxy.MAX_CACHE_MISSES_BEFORE_RELOAD)); + + mockListShardsForSingleResponse(updatedShards); + + Shard shard = proxy.getShard(NOT_CACHED_SHARD); + assertThat(shard, notNullValue()); + assertThat(proxy.getCacheMisses().get(), equalTo(0)); + verify(mockClient).listShards(any()); + + } + + @Test + public void testGetShardCacheMissesResetsAfterLoadAfterMiss() { + proxy.setCachedShardMap(shardMap); + proxy.setCacheMisses(new AtomicInteger(KinesisProxy.MAX_CACHE_MISSES_BEFORE_RELOAD)); + + when(mockClient.listShards(any())).thenReturn(listShardsResult); + when(listShardsResult.getShards()).thenReturn(shards); + when(listShardsResult.getNextToken()).thenReturn(null); + + Shard shard = proxy.getShard(NOT_CACHED_SHARD); + assertThat(shard, nullValue()); + assertThat(proxy.getCacheMisses().get(), equalTo(0)); + } + + @Test + public void testGetShardCacheUpdatedFromAge() { + Instant lastUpdateTime = Instant.now().minus(KinesisProxy.CACHE_MAX_ALLOWED_AGE).minus(KinesisProxy.CACHE_MAX_ALLOWED_AGE); + proxy.setCachedShardMap(shardMap); + proxy.setLastCacheUpdateTime(lastUpdateTime); + + mockListShardsForSingleResponse(updatedShards); + + Shard shard = proxy.getShard(NOT_CACHED_SHARD); + assertThat(shard, notNullValue()); + assertThat(shard.getShardId(), equalTo(NOT_CACHED_SHARD)); + + assertThat(proxy.getLastCacheUpdateTime(), not(equalTo(lastUpdateTime))); + verify(mockClient).listShards(any()); + } + + @Test + public void testGetShardCacheNotUpdatedIfNotOldEnough() { + Instant lastUpdateTime = Instant.now().minus(KinesisProxy.CACHE_MAX_ALLOWED_AGE.toMillis() / 2, ChronoUnit.MILLIS); + proxy.setCachedShardMap(shardMap); + proxy.setLastCacheUpdateTime(lastUpdateTime); + + Shard shard = proxy.getShard(NOT_CACHED_SHARD); + assertThat(shard, nullValue()); + + assertThat(proxy.getLastCacheUpdateTime(), equalTo(lastUpdateTime)); + verify(mockClient, never()).listShards(any()); + } + + @Test + public void testGetShardCacheAgeEmptyForcesUpdate() { + proxy.setCachedShardMap(shardMap); + + mockListShardsForSingleResponse(updatedShards); + Shard shard = proxy.getShard(NOT_CACHED_SHARD); + + assertThat(shard, notNullValue()); + assertThat(shard.getShardId(), equalTo(NOT_CACHED_SHARD)); + + verify(mockClient).listShards(any()); + } + + private void mockListShardsForSingleResponse(List shards) { + when(mockClient.listShards(any())).thenReturn(listShardsResult); + when(listShardsResult.getShards()).thenReturn(shards); + when(listShardsResult.getNextToken()).thenReturn(null); + } + + private DescribeStreamResult createGetStreamInfoResponse(List shards1, boolean isHasMoreShards) { // Create stream description StreamDescription description = new StreamDescription(); @@ -145,14 +503,53 @@ public class KinesisProxyTest { return response; } - // Matcher for testing describe stream request with specific start shard ID. - private static class IsRequestWithStartShardId extends ArgumentMatcher { + private IsRequestWithStartShardId describeWithoutShardId() { + return describeWithShardId(null); + } + + private IsRequestWithStartShardId describeWithShardId(String shardId) { + return new IsRequestWithStartShardId(shardId); + } + + private static class IsRequestWithStartShardId extends TypeSafeDiagnosingMatcher { + private final String shardId; public IsRequestWithStartShardId(String shardId) { this.shardId = shardId; } + @Override + protected boolean matchesSafely(DescribeStreamRequest item, Description mismatchDescription) { + if (shardId == null) { + if (item.getExclusiveStartShardId() != null) { + mismatchDescription.appendText("Expected starting shard id of null, but was ") + .appendValue(item.getExclusiveStartShardId()); + return false; + } + } else if (!shardId.equals(item.getExclusiveStartShardId())) { + mismatchDescription.appendValue(shardId).appendText(" doesn't match expected ") + .appendValue(item.getExclusiveStartShardId()); + return false; + } + + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("A DescribeStreamRequest with a starting shard if of ").appendValue(shardId); + } + } + // Matcher for testing describe stream request with specific start shard ID. + + private static class OldIsRequestWithStartShardId extends ArgumentMatcher { + private final String shardId; + + public OldIsRequestWithStartShardId(String shardId) { + this.shardId = shardId; + } + @Override public boolean matches(Object request) { String startShardId = ((DescribeStreamRequest) request).getExclusiveStartShardId(); @@ -164,4 +561,57 @@ public class KinesisProxyTest { } } + private static ListShardsRequestMatcher initialListShardsRequestMatcher() { + return new ListShardsRequestMatcher(null, null); + } + + private static ListShardsRequestMatcher listShardsNextToken(final String nextToken) { + return new ListShardsRequestMatcher(null, nextToken); + } + + @AllArgsConstructor + private static class ListShardsRequestMatcher extends TypeSafeDiagnosingMatcher { + private final String shardId; + private final String nextToken; + + @Override + protected boolean matchesSafely(final ListShardsRequest listShardsRequest, final Description description) { + if (shardId == null) { + if (StringUtils.isNotEmpty(listShardsRequest.getExclusiveStartShardId())) { + description.appendText("Expected ExclusiveStartShardId to be null, but was ") + .appendValue(listShardsRequest.getExclusiveStartShardId()); + return false; + } + } else { + if (!shardId.equals(listShardsRequest.getExclusiveStartShardId())) { + description.appendText("Expected shardId: ").appendValue(shardId) + .appendText(" doesn't match actual shardId: ") + .appendValue(listShardsRequest.getExclusiveStartShardId()); + return false; + } + } + + if (StringUtils.isNotEmpty(listShardsRequest.getNextToken())) { + if (StringUtils.isNotEmpty(listShardsRequest.getStreamName()) || StringUtils.isNotEmpty(listShardsRequest.getExclusiveStartShardId())) { + return false; + } + + if (!listShardsRequest.getNextToken().equals(nextToken)) { + description.appendText("Found nextToken: ").appendValue(listShardsRequest.getNextToken()) + .appendText(" when it was supposed to be null."); + return false; + } + } else { + return nextToken == null; + } + return true; + } + + @Override + public void describeTo(final Description description) { + description.appendText("A ListShardsRequest with a shardId: ").appendValue(shardId) + .appendText(" and empty nextToken"); + } + } + } diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/types/ShutdownReasonTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/types/ShutdownReasonTest.java index 011e0721..0b9a72f1 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/types/ShutdownReasonTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/types/ShutdownReasonTest.java @@ -1,3 +1,17 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ package com.amazonaws.services.kinesis.clientlibrary.types; import static org.hamcrest.CoreMatchers.equalTo; diff --git a/src/test/java/com/amazonaws/services/kinesis/leases/impl/KinesisClientLeaseBuilder.java b/src/test/java/com/amazonaws/services/kinesis/leases/impl/KinesisClientLeaseBuilder.java index df39b9f2..2e8879fe 100644 --- a/src/test/java/com/amazonaws/services/kinesis/leases/impl/KinesisClientLeaseBuilder.java +++ b/src/test/java/com/amazonaws/services/kinesis/leases/impl/KinesisClientLeaseBuilder.java @@ -1,3 +1,17 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ package com.amazonaws.services.kinesis.leases.impl; import java.util.HashSet; @@ -13,6 +27,7 @@ public class KinesisClientLeaseBuilder { private UUID concurrencyToken; private Long lastCounterIncrementNanos; private ExtendedSequenceNumber checkpoint; + private ExtendedSequenceNumber pendingCheckpoint; private Long ownerSwitchesSinceCheckpoint = 0L; private Set parentShardIds = new HashSet<>(); @@ -46,6 +61,11 @@ public class KinesisClientLeaseBuilder { return this; } + public KinesisClientLeaseBuilder withPendingCheckpoint(ExtendedSequenceNumber pendingCheckpoint) { + this.pendingCheckpoint = pendingCheckpoint; + return this; + } + public KinesisClientLeaseBuilder withOwnerSwitchesSinceCheckpoint(Long ownerSwitchesSinceCheckpoint) { this.ownerSwitchesSinceCheckpoint = ownerSwitchesSinceCheckpoint; return this; @@ -58,6 +78,6 @@ public class KinesisClientLeaseBuilder { public KinesisClientLease build() { return new KinesisClientLease(leaseKey, leaseOwner, leaseCounter, concurrencyToken, lastCounterIncrementNanos, - checkpoint, ownerSwitchesSinceCheckpoint, parentShardIds); + checkpoint, pendingCheckpoint, ownerSwitchesSinceCheckpoint, parentShardIds); } } \ No newline at end of file diff --git a/src/test/java/com/amazonaws/services/kinesis/leases/impl/LeaseIntegrationTest.java b/src/test/java/com/amazonaws/services/kinesis/leases/impl/LeaseIntegrationTest.java index 57a9c99b..e7ff0ebe 100644 --- a/src/test/java/com/amazonaws/services/kinesis/leases/impl/LeaseIntegrationTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/leases/impl/LeaseIntegrationTest.java @@ -1,21 +1,22 @@ /* - * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.leases.impl; import java.util.logging.Logger; +import com.amazonaws.services.kinesis.leases.exceptions.LeasingException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.junit.Ignore; diff --git a/src/test/java/com/amazonaws/services/kinesis/leases/impl/LeaseManagerIntegrationTest.java b/src/test/java/com/amazonaws/services/kinesis/leases/impl/LeaseManagerIntegrationTest.java index 23cc9fc1..dcaedc38 100644 --- a/src/test/java/com/amazonaws/services/kinesis/leases/impl/LeaseManagerIntegrationTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/leases/impl/LeaseManagerIntegrationTest.java @@ -1,16 +1,16 @@ /* - * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.leases.impl; @@ -108,7 +108,8 @@ public class LeaseManagerIntegrationTest extends LeaseIntegrationTest { KinesisClientLease leaseCopy = leaseManager.getLease(lease.getLeaseKey()); - leaseManager.renewLease(lease); + // lose lease + leaseManager.takeLease(lease, "bar"); Assert.assertFalse(leaseManager.renewLease(leaseCopy)); } diff --git a/src/test/java/com/amazonaws/services/kinesis/leases/impl/LeaseRenewerIntegrationTest.java b/src/test/java/com/amazonaws/services/kinesis/leases/impl/LeaseRenewerIntegrationTest.java index 9792d006..8ad19d34 100644 --- a/src/test/java/com/amazonaws/services/kinesis/leases/impl/LeaseRenewerIntegrationTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/leases/impl/LeaseRenewerIntegrationTest.java @@ -1,30 +1,29 @@ /* - * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.leases.impl; -import java.util.Collections; -import java.util.Map; -import java.util.concurrent.Executors; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber; import com.amazonaws.services.kinesis.leases.exceptions.LeasingException; import com.amazonaws.services.kinesis.leases.interfaces.ILeaseRenewer; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.Executors; public class LeaseRenewerIntegrationTest extends LeaseIntegrationTest { @@ -58,7 +57,9 @@ public class LeaseRenewerIntegrationTest extends LeaseIntegrationTest { builder.addLeasesToRenew(renewer, "1", "2"); KinesisClientLease renewedLease = builder.renewMutateAssert(renewer, "1", "2").get("2"); - leaseManager.updateLease(renewedLease); + // lose lease 2 + leaseManager.takeLease(renewedLease, "bar"); + builder.renewMutateAssert(renewer, "1"); } @@ -96,9 +97,9 @@ public class LeaseRenewerIntegrationTest extends LeaseIntegrationTest { public void testGetCurrentlyHeldLeases() throws LeasingException { TestHarnessBuilder builder = new TestHarnessBuilder(leaseManager); - KinesisClientLease lease2 = builder.withLease("1", "foo").withLease("2", "foo").build().get("2"); + builder.withLease("1", "foo").withLease("2", "foo").build(); builder.addLeasesToRenew(renewer, "1", "2"); - builder.renewMutateAssert(renewer, "1", "2"); + KinesisClientLease lease2 = builder.renewMutateAssert(renewer, "1", "2").get("2"); // This should be a copy that doesn't get updated Map heldLeases = renewer.getCurrentlyHeldLeases(); @@ -106,7 +107,9 @@ public class LeaseRenewerIntegrationTest extends LeaseIntegrationTest { Assert.assertEquals((Long) 1L, heldLeases.get("1").getLeaseCounter()); Assert.assertEquals((Long) 1L, heldLeases.get("2").getLeaseCounter()); - leaseManager.updateLease(lease2); // lose lease 2 + // lose lease 2 + leaseManager.takeLease(lease2, "bar"); + // Do another renewal and make sure the copy doesn't change builder.renewMutateAssert(renewer, "1"); @@ -176,7 +179,7 @@ public class LeaseRenewerIntegrationTest extends LeaseIntegrationTest { KinesisClientLease lease = renewer.getCurrentlyHeldLease("1"); // cause lease loss such that the renewer knows the lease has been lost when update is called - leaseManager.renewLease(lease); + leaseManager.takeLease(lease, "bar"); builder.renewMutateAssert(renewer); lease.setCheckpoint(new ExtendedSequenceNumber("new checkpoint")); @@ -195,7 +198,7 @@ public class LeaseRenewerIntegrationTest extends LeaseIntegrationTest { KinesisClientLease lease = renewer.getCurrentlyHeldLease("1"); // cause lease loss such that the renewer knows the lease has been lost when update is called - leaseManager.renewLease(lease); + leaseManager.takeLease(lease, "bar"); builder.renewMutateAssert(renewer); // regain the lease diff --git a/src/test/java/com/amazonaws/services/kinesis/leases/impl/TestHarnessBuilder.java b/src/test/java/com/amazonaws/services/kinesis/leases/impl/TestHarnessBuilder.java index 6b6d673c..0dfbb568 100644 --- a/src/test/java/com/amazonaws/services/kinesis/leases/impl/TestHarnessBuilder.java +++ b/src/test/java/com/amazonaws/services/kinesis/leases/impl/TestHarnessBuilder.java @@ -1,16 +1,16 @@ /* - * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.leases.impl; @@ -35,6 +35,7 @@ public class TestHarnessBuilder { private Map leases = new HashMap(); private KinesisClientLeaseManager leaseManager; + private Map originalLeases = new HashMap<>(); private Callable timeProvider = new Callable() { @@ -54,6 +55,15 @@ public class TestHarnessBuilder { } public TestHarnessBuilder withLease(String shardId, String owner) { + KinesisClientLease lease = createLease(shardId, owner); + KinesisClientLease originalLease = createLease(shardId, owner); + + leases.put(shardId, lease); + originalLeases.put(shardId, originalLease); + return this; + } + + private KinesisClientLease createLease(String shardId, String owner) { KinesisClientLease lease = new KinesisClientLease(); lease.setCheckpoint(new ExtendedSequenceNumber("checkpoint")); lease.setOwnerSwitchesSinceCheckpoint(0L); @@ -62,8 +72,7 @@ public class TestHarnessBuilder { lease.setParentShardIds(Collections.singleton("parentShardId")); lease.setLeaseKey(shardId); - leases.put(shardId, lease); - return this; + return lease; } public Map build() throws LeasingException { @@ -147,7 +156,7 @@ public class TestHarnessBuilder { Assert.assertEquals(renewedShardIds.length, heldLeases.size()); for (String shardId : renewedShardIds) { - KinesisClientLease original = leases.get(shardId); + KinesisClientLease original = originalLeases.get(shardId); Assert.assertNotNull(original); KinesisClientLease actual = heldLeases.get(shardId); diff --git a/src/test/java/com/amazonaws/services/kinesis/multilang/Matchers.java b/src/test/java/com/amazonaws/services/kinesis/multilang/Matchers.java index b84d61a0..92ac15f7 100644 --- a/src/test/java/com/amazonaws/services/kinesis/multilang/Matchers.java +++ b/src/test/java/com/amazonaws/services/kinesis/multilang/Matchers.java @@ -1,3 +1,17 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ package com.amazonaws.services.kinesis.multilang; import static org.hamcrest.CoreMatchers.equalTo; diff --git a/src/test/java/com/amazonaws/services/kinesis/multilang/MessageWriterTest.java b/src/test/java/com/amazonaws/services/kinesis/multilang/MessageWriterTest.java index 08f04c92..f9fd1d58 100644 --- a/src/test/java/com/amazonaws/services/kinesis/multilang/MessageWriterTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/multilang/MessageWriterTest.java @@ -1,16 +1,16 @@ /* - * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.multilang; @@ -113,6 +113,16 @@ public class MessageWriterTest { Mockito.verify(this.stream, Mockito.atLeastOnce()).flush(); } + @Test + public void writeShutdownRequestedMessageTest() throws IOException, InterruptedException, ExecutionException { + Future future = this.messageWriter.writeShutdownRequestedMessage(); + future.get(); + + Mockito.verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), + Mockito.anyInt()); + Mockito.verify(this.stream, Mockito.atLeastOnce()).flush(); + } + @Test public void streamIOExceptionTest() throws IOException, InterruptedException, ExecutionException { Mockito.doThrow(IOException.class).when(stream).flush(); diff --git a/src/test/java/com/amazonaws/services/kinesis/multilang/MultiLangDaemonConfigTest.java b/src/test/java/com/amazonaws/services/kinesis/multilang/MultiLangDaemonConfigTest.java index 6a687577..1bd3580d 100644 --- a/src/test/java/com/amazonaws/services/kinesis/multilang/MultiLangDaemonConfigTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/multilang/MultiLangDaemonConfigTest.java @@ -18,6 +18,10 @@ import static org.junit.Assert.assertNotNull; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.lang.reflect.Field; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import java.util.Properties; import junit.framework.Assert; @@ -39,23 +43,38 @@ public class MultiLangDaemonConfigTest { AWSCredentials creds = Mockito.mock(AWSCredentials.class); Mockito.doReturn(creds).when(credentialsProvider).getCredentials(); Mockito.doReturn("cool-user").when(creds).getAWSAccessKeyId(); - KinesisClientLibConfiguration kclConfig = - new KinesisClientLibConfiguration("cool-app", "cool-stream", credentialsProvider, "cool-worker"); + KinesisClientLibConfiguration kclConfig = new KinesisClientLibConfiguration("cool-app", "cool-stream", + credentialsProvider, "cool-worker"); KinesisClientLibConfigurator configurator = Mockito.mock(KinesisClientLibConfigurator.class); Mockito.doReturn(kclConfig).when(configurator).getConfiguration(Mockito.any(Properties.class)); return configurator; } + // Function to mock ENV variables + private void setEnv(Map newenv) throws Exception { + Class[] classes = Collections.class.getDeclaredClasses(); + Map env = System.getenv(); + for (Class cl : classes) { + if ("java.util.Collections$UnmodifiableMap".equals(cl.getName())) { + Field field = cl.getDeclaredField("m"); + field.setAccessible(true); + Object obj = field.get(env); + Map map = (Map) obj; + map.clear(); + map.putAll(newenv); + } + } + } + @Test public void constructorTest() throws IOException { - String PROPERTIES = + String properties = "executableName = randomEXE \n" + "applicationName = testApp \n" + "streamName = fakeStream \n" + "AWSCredentialsProvider = DefaultAWSCredentialsProviderChain\n" + "processingLanguage = malbolge"; ClassLoader classLoader = Mockito.mock(ClassLoader.class); - Mockito.doReturn(new ByteArrayInputStream(PROPERTIES.getBytes())) - .when(classLoader) + Mockito.doReturn(new ByteArrayInputStream(properties.getBytes())).when(classLoader) .getResourceAsStream(FILENAME); MultiLangDaemonConfig deamonConfig = new MultiLangDaemonConfig(FILENAME, classLoader, buildMockConfigurator()); @@ -67,14 +86,11 @@ public class MultiLangDaemonConfigTest { @Test public void propertyValidation() { - String PROPERTIES_NO_EXECUTABLE_NAME = - "applicationName = testApp \n" + "streamName = fakeStream \n" - + "AWSCredentialsProvider = DefaultAWSCredentialsProviderChain\n" - + "processingLanguage = malbolge"; + String propertiesNoExecutableName = "applicationName = testApp \n" + "streamName = fakeStream \n" + + "AWSCredentialsProvider = DefaultAWSCredentialsProviderChain\n" + "processingLanguage = malbolge"; ClassLoader classLoader = Mockito.mock(ClassLoader.class); - Mockito.doReturn(new ByteArrayInputStream(PROPERTIES_NO_EXECUTABLE_NAME.getBytes())) - .when(classLoader) + Mockito.doReturn(new ByteArrayInputStream(propertiesNoExecutableName.getBytes())).when(classLoader) .getResourceAsStream(FILENAME); MultiLangDaemonConfig config; @@ -88,4 +104,87 @@ public class MultiLangDaemonConfigTest { } } + @Test + public void testKinesisClientLibConfigurationShouldGetProxyInfoFromPropertiesFile() throws Exception { + String properties = + "executableName = randomEXE \n" + "applicationName = testApp \n" + "streamName = fakeStream \n" + + "AWSCredentialsProvider = DefaultAWSCredentialsProviderChain\n" + + "http.proxyHost = http://proxy.com\n" + "http.proxyPort = 1234\n" + + "processingLanguage = malbolge"; + ClassLoader classLoader = Mockito.mock(ClassLoader.class); + + Mockito.doReturn(new ByteArrayInputStream(properties.getBytes())).when(classLoader) + .getResourceAsStream(FILENAME); + + MultiLangDaemonConfig config = new MultiLangDaemonConfig(FILENAME, classLoader, buildMockConfigurator()); + assertAgainstKclConfig(config.getKinesisClientLibConfiguration(), "http://proxy.com", 1234); + } + + @Test + public void testKinesisClientLibConfigurationShouldGetProxyInfoFromSystemProperties() throws Exception { + String properties = + "executableName = randomEXE \n" + "applicationName = testApp \n" + "streamName = fakeStream \n" + + "AWSCredentialsProvider = DefaultAWSCredentialsProviderChain\n" + + "processingLanguage = malbolge"; + ClassLoader classLoader = Mockito.mock(ClassLoader.class); + + Mockito.doReturn(new ByteArrayInputStream(properties.getBytes())).when(classLoader) + .getResourceAsStream(FILENAME); + + System.setProperty(MultiLangDaemonConfig.PROXY_HOST_PROP, "http://proxy.com"); + System.setProperty(MultiLangDaemonConfig.PROXY_PORT_PROP, "1234"); + + MultiLangDaemonConfig config = new MultiLangDaemonConfig(FILENAME, classLoader, buildMockConfigurator()); + assertAgainstKclConfig(config.getKinesisClientLibConfiguration(), "http://proxy.com", 1234); + } + + @Test + public void testKinesisClientLibConfigurationShouldGetProxyInfoFromEnvVars() throws Exception { + String properties = + "executableName = randomEXE \n" + "applicationName = testApp \n" + "streamName = fakeStream \n" + + "AWSCredentialsProvider = DefaultAWSCredentialsProviderChain\n" + + "processingLanguage = malbolge"; + ClassLoader classLoader = Mockito.mock(ClassLoader.class); + + Map env = new HashMap<>(); + env.put(MultiLangDaemonConfig.HTTP_PROXY_ENV_VAR, "http://proxy.com:1234"); + + setEnv(env); + + Mockito.doReturn(new ByteArrayInputStream(properties.getBytes())).when(classLoader) + .getResourceAsStream(FILENAME); + + MultiLangDaemonConfig config = new MultiLangDaemonConfig(FILENAME, classLoader, buildMockConfigurator()); + + assertAgainstKclConfig(config.getKinesisClientLibConfiguration(), "http://proxy.com", 1234); + } + + @Test + public void testKinesisClientLibConfigurationShouldNotGetProxyInfo() throws Exception { + String properties = + "executableName = randomEXE \n" + "applicationName = testApp \n" + "streamName = fakeStream \n" + + "AWSCredentialsProvider = DefaultAWSCredentialsProviderChain\n" + + "processingLanguage = malbolge"; + ClassLoader classLoader = Mockito.mock(ClassLoader.class); + + Map env = new HashMap<>(); + + //clear out any env vars loaded from system + setEnv(env); + + Mockito.doReturn(new ByteArrayInputStream(properties.getBytes())).when(classLoader) + .getResourceAsStream(FILENAME); + + MultiLangDaemonConfig config = new MultiLangDaemonConfig(FILENAME, classLoader, buildMockConfigurator()); + assertAgainstKclConfig(config.getKinesisClientLibConfiguration(), null, -1); + } + + private void assertAgainstKclConfig(KinesisClientLibConfiguration kclConfig, String host, int port) { + Assert.assertEquals(host, kclConfig.getKinesisClientConfiguration().getProxyHost()); + Assert.assertEquals(host, kclConfig.getDynamoDBClientConfiguration().getProxyHost()); + Assert.assertEquals(host, kclConfig.getCloudWatchClientConfiguration().getProxyHost()); + Assert.assertEquals(port, kclConfig.getKinesisClientConfiguration().getProxyPort()); + Assert.assertEquals(port, kclConfig.getDynamoDBClientConfiguration().getProxyPort()); + Assert.assertEquals(port, kclConfig.getCloudWatchClientConfiguration().getProxyPort()); + } } diff --git a/src/test/java/com/amazonaws/services/kinesis/multilang/MultiLangProtocolTest.java b/src/test/java/com/amazonaws/services/kinesis/multilang/MultiLangProtocolTest.java index f00bb48f..da14d256 100644 --- a/src/test/java/com/amazonaws/services/kinesis/multilang/MultiLangProtocolTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/multilang/MultiLangProtocolTest.java @@ -1,77 +1,91 @@ /* - * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.multilang; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.junit.Assert.assertThat; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyLong; -import static org.mockito.Matchers.anyString; -import static org.mockito.Matchers.argThat; -import static org.mockito.Mockito.timeout; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; - -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mockito; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; - import com.amazonaws.services.kinesis.clientlibrary.exceptions.InvalidStateException; import com.amazonaws.services.kinesis.clientlibrary.exceptions.KinesisClientLibDependencyException; import com.amazonaws.services.kinesis.clientlibrary.exceptions.ShutdownException; import com.amazonaws.services.kinesis.clientlibrary.exceptions.ThrottlingException; import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer; - +import com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisClientLibConfiguration; +import com.amazonaws.services.kinesis.clientlibrary.lib.worker.ShutdownReason; import com.amazonaws.services.kinesis.clientlibrary.types.InitializationInput; import com.amazonaws.services.kinesis.clientlibrary.types.ProcessRecordsInput; -import com.amazonaws.services.kinesis.clientlibrary.lib.worker.ShutdownReason; - import com.amazonaws.services.kinesis.model.Record; import com.amazonaws.services.kinesis.multilang.messages.CheckpointMessage; import com.amazonaws.services.kinesis.multilang.messages.Message; import com.amazonaws.services.kinesis.multilang.messages.ProcessRecordsMessage; import com.amazonaws.services.kinesis.multilang.messages.StatusMessage; import com.google.common.util.concurrent.SettableFuture; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.runners.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.argThat; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) public class MultiLangProtocolTest { private static final List EMPTY_RECORD_LIST = Collections.emptyList(); + @Mock private MultiLangProtocol protocol; + @Mock private MessageWriter messageWriter; + @Mock private MessageReader messageReader; private String shardId; + @Mock private IRecordProcessorCheckpointer checkpointer; + @Mock + private KinesisClientLibConfiguration configuration; @Before public void setup() { this.shardId = "shard-id-123"; - messageWriter = Mockito.mock(MessageWriter.class); - messageReader = Mockito.mock(MessageReader.class); - protocol = new MultiLangProtocol(messageReader, messageWriter, new InitializationInput().withShardId(shardId)); - checkpointer = Mockito.mock(IRecordProcessorCheckpointer.class); + protocol = new MultiLangProtocolForTesting(messageReader, messageWriter, + new InitializationInput().withShardId(shardId), configuration); + + when(configuration.getTimeoutInSeconds()).thenReturn(Optional.empty()); } private Future buildFuture(T value) { @@ -114,6 +128,16 @@ public class MultiLangProtocolTest { assertThat(protocol.shutdown(null, ShutdownReason.ZOMBIE), equalTo(true)); } + @Test + public void shutdownRequestedTest() { + when(messageWriter.writeShutdownRequestedMessage()).thenReturn(buildFuture(true)); + when(messageReader.getNextMessageFromSTDOUT()).thenReturn(buildFuture(new StatusMessage("shutdownRequested"), Message.class)); + Mockito.doReturn(buildFuture(true)).when(messageWriter) + .writeShutdownRequestedMessage(); + Mockito.doReturn(buildFuture(new StatusMessage("shutdownRequested"))).when(messageReader).getNextMessageFromSTDOUT(); + assertThat(protocol.shutdownRequested(null), equalTo(true)); + } + private Answer> buildMessageAnswers(List messages) { return new Answer>() { @@ -155,7 +179,10 @@ public class MultiLangProtocolTest { this.add(new StatusMessage("processRecords")); } })); - assertThat(protocol.processRecords(new ProcessRecordsInput().withRecords(EMPTY_RECORD_LIST).withCheckpointer(checkpointer)), equalTo(true)); + + boolean result = protocol.processRecords(new ProcessRecordsInput().withRecords(EMPTY_RECORD_LIST).withCheckpointer(checkpointer)); + + assertThat(result, equalTo(true)); verify(checkpointer, timeout(1)).checkpoint(); verify(checkpointer, timeout(1)).checkpoint("123", 0L); @@ -173,4 +200,50 @@ public class MultiLangProtocolTest { })); assertThat(protocol.processRecords(new ProcessRecordsInput().withRecords(EMPTY_RECORD_LIST).withCheckpointer(checkpointer)), equalTo(false)); } + + @Test(expected = NullPointerException.class) + public void waitForStatusMessageTimeoutTest() throws InterruptedException, TimeoutException, ExecutionException { + when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class))).thenReturn(buildFuture(true)); + Future future = Mockito.mock(Future.class); + when(messageReader.getNextMessageFromSTDOUT()).thenReturn(future); + when(configuration.getTimeoutInSeconds()).thenReturn(Optional.of(5)); + when(future.get(anyInt(), eq(TimeUnit.SECONDS))).thenThrow(TimeoutException.class); + protocol = new MultiLangProtocolForTesting(messageReader, + messageWriter, + new InitializationInput().withShardId(shardId), + configuration); + + protocol.processRecords(new ProcessRecordsInput().withRecords(EMPTY_RECORD_LIST)); + } + + @Test + public void waitForStatusMessageSuccessTest() { + when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class))).thenReturn(buildFuture(true)); + when(messageReader.getNextMessageFromSTDOUT()).thenReturn(buildFuture(new StatusMessage("processRecords"), Message.class)); + when(configuration.getTimeoutInSeconds()).thenReturn(Optional.of(5)); + + assertTrue(protocol.processRecords(new ProcessRecordsInput().withRecords(EMPTY_RECORD_LIST))); + } + + private class MultiLangProtocolForTesting extends MultiLangProtocol { + /** + * Constructor. + * + * @param messageReader A message reader. + * @param messageWriter A message writer. + * @param initializationInput + * @param configuration + */ + MultiLangProtocolForTesting(final MessageReader messageReader, + final MessageWriter messageWriter, + final InitializationInput initializationInput, + final KinesisClientLibConfiguration configuration) { + super(messageReader, messageWriter, initializationInput, configuration); + } + + @Override + protected void haltJvm(final int exitStatus) { + throw new NullPointerException(); + } + } } diff --git a/src/test/java/com/amazonaws/services/kinesis/multilang/StreamingRecordProcessorFactoryTest.java b/src/test/java/com/amazonaws/services/kinesis/multilang/StreamingRecordProcessorFactoryTest.java index a8f5885b..ba3e735b 100644 --- a/src/test/java/com/amazonaws/services/kinesis/multilang/StreamingRecordProcessorFactoryTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/multilang/StreamingRecordProcessorFactoryTest.java @@ -1,29 +1,37 @@ /* - * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.multilang; +import com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisClientLibConfiguration; import org.junit.Assert; import org.junit.Test; import com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessor; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; +@RunWith(MockitoJUnitRunner.class) public class StreamingRecordProcessorFactoryTest { + @Mock + private KinesisClientLibConfiguration configuration; + @Test public void createProcessorTest() { - MultiLangRecordProcessorFactory factory = new MultiLangRecordProcessorFactory("somecommand", null); + MultiLangRecordProcessorFactory factory = new MultiLangRecordProcessorFactory("somecommand", null, configuration); IRecordProcessor processor = factory.createProcessor(); Assert.assertEquals("Should have constructed a StreamingRecordProcessor", MultiLangRecordProcessor.class, diff --git a/src/test/java/com/amazonaws/services/kinesis/multilang/StreamingRecordProcessorTest.java b/src/test/java/com/amazonaws/services/kinesis/multilang/StreamingRecordProcessorTest.java index 2c02b5e9..f32fa5bf 100644 --- a/src/test/java/com/amazonaws/services/kinesis/multilang/StreamingRecordProcessorTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/multilang/StreamingRecordProcessorTest.java @@ -1,54 +1,27 @@ /* - * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.multilang; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyLong; -import static org.mockito.Matchers.anyString; -import static org.mockito.Matchers.argThat; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - -import com.amazonaws.services.kinesis.clientlibrary.lib.worker.ShutdownReason; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.runners.MockitoJUnitRunner; -import org.mockito.stubbing.Answer; - import com.amazonaws.services.kinesis.clientlibrary.exceptions.InvalidStateException; import com.amazonaws.services.kinesis.clientlibrary.exceptions.KinesisClientLibDependencyException; import com.amazonaws.services.kinesis.clientlibrary.exceptions.ShutdownException; import com.amazonaws.services.kinesis.clientlibrary.exceptions.ThrottlingException; +import com.amazonaws.services.kinesis.clientlibrary.interfaces.IPreparedCheckpointer; import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer; +import com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisClientLibConfiguration; +import com.amazonaws.services.kinesis.clientlibrary.lib.worker.ShutdownReason; import com.amazonaws.services.kinesis.clientlibrary.types.InitializationInput; import com.amazonaws.services.kinesis.clientlibrary.types.ProcessRecordsInput; import com.amazonaws.services.kinesis.clientlibrary.types.ShutdownInput; @@ -59,6 +32,35 @@ import com.amazonaws.services.kinesis.multilang.messages.ProcessRecordsMessage; import com.amazonaws.services.kinesis.multilang.messages.ShutdownMessage; import com.amazonaws.services.kinesis.multilang.messages.StatusMessage; import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.runners.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.argThat; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) public class StreamingRecordProcessorTest { @@ -98,6 +100,34 @@ public class StreamingRecordProcessorTest { IllegalArgumentException { throw new UnsupportedOperationException(); } + + @Override + public IPreparedCheckpointer prepareCheckpoint() + throws KinesisClientLibDependencyException, + InvalidStateException, ThrottlingException, ShutdownException { + throw new UnsupportedOperationException(); + } + + @Override + public IPreparedCheckpointer prepareCheckpoint(Record record) + throws KinesisClientLibDependencyException, + InvalidStateException, ThrottlingException, ShutdownException { + throw new UnsupportedOperationException(); + } + + @Override + public IPreparedCheckpointer prepareCheckpoint(String sequenceNumber) + throws KinesisClientLibDependencyException, + InvalidStateException, ThrottlingException, ShutdownException, IllegalArgumentException { + throw new UnsupportedOperationException(); + } + + @Override + public IPreparedCheckpointer prepareCheckpoint(String sequenceNumber, long subSequenceNumber) + throws KinesisClientLibDependencyException, + InvalidStateException, ThrottlingException, ShutdownException, IllegalArgumentException { + throw new UnsupportedOperationException(); + } }; private MessageWriter messageWriter; @@ -108,6 +138,9 @@ public class StreamingRecordProcessorTest { private MultiLangRecordProcessor recordProcessor; + @Mock + private KinesisClientLibConfiguration configuration; + @Before public void prepare() throws IOException, InterruptedException, ExecutionException { // Fake command @@ -121,10 +154,11 @@ public class StreamingRecordProcessorTest { messageWriter = Mockito.mock(MessageWriter.class); messageReader = Mockito.mock(MessageReader.class); errorReader = Mockito.mock(DrainChildSTDERRTask.class); + when(configuration.getTimeoutInSeconds()).thenReturn(Optional.empty()); recordProcessor = new MultiLangRecordProcessor(new ProcessBuilder(), executor, new ObjectMapper(), messageWriter, - messageReader, errorReader) { + messageReader, errorReader, configuration) { // Just don't do anything when we exit. void exit() { diff --git a/src/test/java/com/amazonaws/services/kinesis/multilang/messages/MessageTest.java b/src/test/java/com/amazonaws/services/kinesis/multilang/messages/MessageTest.java index 2c76aa30..2b2fe402 100644 --- a/src/test/java/com/amazonaws/services/kinesis/multilang/messages/MessageTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/multilang/messages/MessageTest.java @@ -1,16 +1,16 @@ /* - * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.multilang.messages; @@ -44,7 +44,7 @@ public class MessageTest { }); } })), new ShutdownMessage(ShutdownReason.ZOMBIE), new StatusMessage("processRecords"), - new InitializeMessage(), new ProcessRecordsMessage() }; + new InitializeMessage(), new ProcessRecordsMessage(), new ShutdownRequestedMessage() }; for (int i = 0; i < messages.length; i++) { Assert.assertTrue("Each message should contain the action field", messages[i].toString().contains("action")); diff --git a/src/test/java/log4j.properties b/src/test/resources/log4j.properties similarity index 100% rename from src/test/java/log4j.properties rename to src/test/resources/log4j.properties