From f0557179f1870c22efa8f415a544563272e84970 Mon Sep 17 00:00:00 2001 From: "Pfifer, Justin" Date: Fri, 19 May 2017 08:09:34 -0700 Subject: [PATCH 1/4] Improved Documentation for IdleTimeBetweenReadsMillis Updated the documentation to indicates that it is only used when GetRecords returns an empty set of records. --- .../worker/KinesisClientLibConfiguration.java | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java index b8218968..422838c3 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java @@ -52,7 +52,8 @@ public class KinesisClientLibConfiguration { public static final int DEFAULT_MAX_RECORDS = 10000; /** - * Idle time between record reads in milliseconds. + * The default value for how long the {@link ShardConsumer} should sleep if no records are returned from the call to + * {@link com.amazonaws.services.kinesis.AmazonKinesis#getRecords(com.amazonaws.services.kinesis.model.GetRecordsRequest)}. */ public static final long DEFAULT_IDLETIME_BETWEEN_READS_MILLIS = 1000L; @@ -790,7 +791,24 @@ public class KinesisClientLibConfiguration { } /** - * @param idleTimeBetweenReadsInMillis Idle time between calls to fetch data from Kinesis + * Controls how long the {@link ShardConsumer} will sleep if no records are returned from the call to + * {@link com.amazonaws.services.kinesis.AmazonKinesis#getRecords(com.amazonaws.services.kinesis.model.GetRecordsRequest)}. + * + * This sleep is only used when no records are returned. If records are returned the {@link ShardConsumer} will + * immediately retrieve the next set of records after the call to + * {@link com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessor#processRecords(com.amazonaws.services.kinesis.clientlibrary.types.ProcessRecordsInput)} + * has returned. Setting this value to high may result in the KCL being unable to catch up. If you are changing this + * value it's recommended that you enable {@link #withCallProcessRecordsEvenForEmptyRecordList(boolean)}, and + * monitor how far behind the records retrieved are by inspecting + * {@link com.amazonaws.services.kinesis.clientlibrary.types.ProcessRecordsInput#getMillisBehindLatest()}, and the + * CloudWatch + * Metric: GetRecords.MillisBehindLatest + * + * @param idleTimeBetweenReadsInMillis + * how long to sleep calls to + * {@link com.amazonaws.services.kinesis.AmazonKinesis#getRecords(com.amazonaws.services.kinesis.model.GetRecordsRequest)} + * when no records are returned. * @return KinesisClientLibConfiguration */ public KinesisClientLibConfiguration withIdleTimeBetweenReadsInMillis(long idleTimeBetweenReadsInMillis) { From ddfd64e40ad5be646b97d5b229dfe5160cff0911 Mon Sep 17 00:00:00 2001 From: "Pfifer, Justin" Date: Fri, 19 May 2017 08:10:54 -0700 Subject: [PATCH 2/4] Update the Documentation for Shutdown to List Checkpoint Requirements When the KCL calls shutdown with the reason TERMINATE the customer's application must checkpoint, or the KCL will not make progress. --- .../interfaces/v2/IRecordProcessor.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/interfaces/v2/IRecordProcessor.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/interfaces/v2/IRecordProcessor.java index 25087f03..bd445ac9 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/interfaces/v2/IRecordProcessor.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/interfaces/v2/IRecordProcessor.java @@ -45,10 +45,16 @@ public interface IRecordProcessor { /** * Invoked by the Amazon Kinesis Client Library to indicate it will no longer send data records to this - * RecordProcessor instance. + * RecordProcessor instance. * - * @param shutdownInput Provides information and capabilities (eg checkpointing) related to shutdown of this record - * processor. + *

Warning

+ * + * When the value of {@link ShutdownInput#getShutdownReason()} is + * {@link com.amazonaws.services.kinesis.clientlibrary.lib.worker.ShutdownReason#TERMINATE} it is required that you + * checkpoint. Failure to do so will result in an IllegalArgumentException, and the KCL no longer making progress. + * + * @param shutdownInput + * Provides information and capabilities (eg checkpointing) related to shutdown of this record processor. */ void shutdown(ShutdownInput shutdownInput); From e12ff541f7fc3774041c3c9a2d60fd3cf172f3c6 Mon Sep 17 00:00:00 2001 From: "Pfifer, Justin" Date: Fri, 19 May 2017 08:29:13 -0700 Subject: [PATCH 3/4] Enable Building Both Source, and JavaDoc jars --- pom.xml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/pom.xml b/pom.xml index 16d09bbf..26304866 100644 --- a/pom.xml +++ b/pom.xml @@ -242,6 +242,35 @@ + + org.apache.maven.plugins + maven-javadoc-plugin + 2.10.3 + + com.amazonaws.services.kinesis.producer.protobuf + + + + attach-javadocs + + jar + + + + + + org.apache.maven.plugins + maven-source-plugin + 3.0.1 + + + attach-sources + + jar + + + + From e121691ac2442e29bf8e63e76d0d5d10983022c3 Mon Sep 17 00:00:00 2001 From: "Pfifer, Justin" Date: Fri, 19 May 2017 08:29:42 -0700 Subject: [PATCH 4/4] Cleanup the Writing of the IdleTimeBetweenMillis Documentation --- .../lib/worker/KinesisClientLibConfiguration.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java index 422838c3..02450cb8 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java @@ -791,12 +791,12 @@ public class KinesisClientLibConfiguration { } /** - * Controls how long the {@link ShardConsumer} will sleep if no records are returned from the call to - * {@link com.amazonaws.services.kinesis.AmazonKinesis#getRecords(com.amazonaws.services.kinesis.model.GetRecordsRequest)}. + * Controls how long the KCL will sleep if no records are returned from Kinesis * - * This sleep is only used when no records are returned. If records are returned the {@link ShardConsumer} will + *

+ * This value is only used when no records are returned; if records are returned, the {@link com.amazonaws.services.kinesis.clientlibrary.lib.worker.ProcessTask} will * immediately retrieve the next set of records after the call to - * {@link com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessor#processRecords(com.amazonaws.services.kinesis.clientlibrary.types.ProcessRecordsInput)} + * {@link com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessor#processRecords(ProcessRecordsInput)} * has returned. Setting this value to high may result in the KCL being unable to catch up. If you are changing this * value it's recommended that you enable {@link #withCallProcessRecordsEvenForEmptyRecordList(boolean)}, and * monitor how far behind the records retrieved are by inspecting @@ -804,11 +804,10 @@ public class KinesisClientLibConfiguration { * CloudWatch * Metric: GetRecords.MillisBehindLatest + *

* * @param idleTimeBetweenReadsInMillis - * how long to sleep calls to - * {@link com.amazonaws.services.kinesis.AmazonKinesis#getRecords(com.amazonaws.services.kinesis.model.GetRecordsRequest)} - * when no records are returned. + * how long to sleep between GetRecords calls when no records are returned. * @return KinesisClientLibConfiguration */ public KinesisClientLibConfiguration withIdleTimeBetweenReadsInMillis(long idleTimeBetweenReadsInMillis) {