run spotless:apply

This commit is contained in:
Nakul Joshi 2024-05-20 11:26:15 -07:00 committed by nakulj
parent 409a8f13cd
commit 18fe49eed0
328 changed files with 14517 additions and 10319 deletions

View file

@ -18,11 +18,9 @@ import java.util.Date;
import java.util.Optional; import java.util.Optional;
import java.util.Set; import java.util.Set;
import org.apache.commons.lang3.Validate;
import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSet;
import lombok.Getter; import lombok.Getter;
import org.apache.commons.lang3.Validate;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.kinesis.checkpoint.ShardRecordProcessorCheckpointer; import software.amazon.kinesis.checkpoint.ShardRecordProcessorCheckpointer;
import software.amazon.kinesis.common.InitialPositionInStream; import software.amazon.kinesis.common.InitialPositionInStream;
@ -119,14 +117,16 @@ public class KinesisClientLibConfiguration {
/** /**
* Metrics dimensions that always will be enabled regardless of the config provided by user. * Metrics dimensions that always will be enabled regardless of the config provided by user.
*/ */
public static final Set<String> METRICS_ALWAYS_ENABLED_DIMENSIONS = ImmutableSet public static final Set<String> METRICS_ALWAYS_ENABLED_DIMENSIONS =
.of(MetricsUtil.OPERATION_DIMENSION_NAME); ImmutableSet.of(MetricsUtil.OPERATION_DIMENSION_NAME);
/** /**
* Allowed dimensions for CloudWatch metrics. By default, worker ID dimension will be disabled. * Allowed dimensions for CloudWatch metrics. By default, worker ID dimension will be disabled.
*/ */
public static final Set<String> DEFAULT_METRICS_ENABLED_DIMENSIONS = ImmutableSet.<String> builder() public static final Set<String> DEFAULT_METRICS_ENABLED_DIMENSIONS = ImmutableSet.<String>builder()
.addAll(METRICS_ALWAYS_ENABLED_DIMENSIONS).add(MetricsUtil.SHARD_ID_DIMENSION_NAME).build(); .addAll(METRICS_ALWAYS_ENABLED_DIMENSIONS)
.add(MetricsUtil.SHARD_ID_DIMENSION_NAME)
.build();
/** /**
* Metrics dimensions that signify all possible dimensions. * Metrics dimensions that signify all possible dimensions.
@ -285,8 +285,8 @@ public class KinesisClientLibConfiguration {
* @param workerId * @param workerId
* Used to distinguish different workers/processes of a Kinesis application * Used to distinguish different workers/processes of a Kinesis application
*/ */
public KinesisClientLibConfiguration(String applicationName, String streamName, public KinesisClientLibConfiguration(
AwsCredentialsProvider credentialsProvider, String workerId) { String applicationName, String streamName, AwsCredentialsProvider credentialsProvider, String workerId) {
this(applicationName, streamName, credentialsProvider, credentialsProvider, credentialsProvider, workerId); this(applicationName, streamName, credentialsProvider, credentialsProvider, credentialsProvider, workerId);
} }
@ -308,16 +308,36 @@ public class KinesisClientLibConfiguration {
* @param workerId * @param workerId
* Used to distinguish different workers/processes of a Kinesis application * Used to distinguish different workers/processes of a Kinesis application
*/ */
public KinesisClientLibConfiguration(String applicationName, String streamName, public KinesisClientLibConfiguration(
AwsCredentialsProvider kinesisCredentialsProvider, AwsCredentialsProvider dynamoDBCredentialsProvider, String applicationName,
AwsCredentialsProvider cloudWatchCredentialsProvider, String workerId) { String streamName,
this(applicationName, streamName, null, null, DEFAULT_INITIAL_POSITION_IN_STREAM, kinesisCredentialsProvider, AwsCredentialsProvider kinesisCredentialsProvider,
dynamoDBCredentialsProvider, cloudWatchCredentialsProvider, DEFAULT_FAILOVER_TIME_MILLIS, workerId, AwsCredentialsProvider dynamoDBCredentialsProvider,
DEFAULT_MAX_RECORDS, DEFAULT_IDLETIME_BETWEEN_READS_MILLIS, AwsCredentialsProvider cloudWatchCredentialsProvider,
DEFAULT_DONT_CALL_PROCESS_RECORDS_FOR_EMPTY_RECORD_LIST, DEFAULT_PARENT_SHARD_POLL_INTERVAL_MILLIS, String workerId) {
DEFAULT_SHARD_SYNC_INTERVAL_MILLIS, DEFAULT_CLEANUP_LEASES_UPON_SHARDS_COMPLETION, this(
DEFAULT_TASK_BACKOFF_TIME_MILLIS, DEFAULT_METRICS_BUFFER_TIME_MILLIS, DEFAULT_METRICS_MAX_QUEUE_SIZE, applicationName,
DEFAULT_VALIDATE_SEQUENCE_NUMBER_BEFORE_CHECKPOINTING, null, DEFAULT_SHUTDOWN_GRACE_MILLIS, streamName,
null,
null,
DEFAULT_INITIAL_POSITION_IN_STREAM,
kinesisCredentialsProvider,
dynamoDBCredentialsProvider,
cloudWatchCredentialsProvider,
DEFAULT_FAILOVER_TIME_MILLIS,
workerId,
DEFAULT_MAX_RECORDS,
DEFAULT_IDLETIME_BETWEEN_READS_MILLIS,
DEFAULT_DONT_CALL_PROCESS_RECORDS_FOR_EMPTY_RECORD_LIST,
DEFAULT_PARENT_SHARD_POLL_INTERVAL_MILLIS,
DEFAULT_SHARD_SYNC_INTERVAL_MILLIS,
DEFAULT_CLEANUP_LEASES_UPON_SHARDS_COMPLETION,
DEFAULT_TASK_BACKOFF_TIME_MILLIS,
DEFAULT_METRICS_BUFFER_TIME_MILLIS,
DEFAULT_METRICS_MAX_QUEUE_SIZE,
DEFAULT_VALIDATE_SEQUENCE_NUMBER_BEFORE_CHECKPOINTING,
null,
DEFAULT_SHUTDOWN_GRACE_MILLIS,
DEFAULT_SCHEDULER_INITIALIZATION_BACKOFF_TIME_MILLIS); DEFAULT_SCHEDULER_INITIALIZATION_BACKOFF_TIME_MILLIS);
} }
@ -377,20 +397,53 @@ public class KinesisClientLibConfiguration {
*/ */
// CHECKSTYLE:IGNORE HiddenFieldCheck FOR NEXT 26 LINES // CHECKSTYLE:IGNORE HiddenFieldCheck FOR NEXT 26 LINES
// CHECKSTYLE:IGNORE ParameterNumber FOR NEXT 26 LINES // CHECKSTYLE:IGNORE ParameterNumber FOR NEXT 26 LINES
public KinesisClientLibConfiguration(String applicationName, String streamName, String kinesisEndpoint, public KinesisClientLibConfiguration(
InitialPositionInStream initialPositionInStream, AwsCredentialsProvider kinesisCredentialsProvider, String applicationName,
AwsCredentialsProvider dynamoDBCredentialsProvider, AwsCredentialsProvider cloudWatchCredentialsProvider, String streamName,
long failoverTimeMillis, String workerId, int maxRecords, long idleTimeBetweenReadsInMillis, String kinesisEndpoint,
boolean callProcessRecordsEvenForEmptyRecordList, long parentShardPollIntervalMillis, InitialPositionInStream initialPositionInStream,
long shardSyncIntervalMillis, boolean cleanupTerminatedShardsBeforeExpiry, long taskBackoffTimeMillis, AwsCredentialsProvider kinesisCredentialsProvider,
long metricsBufferTimeMillis, int metricsMaxQueueSize, boolean validateSequenceNumberBeforeCheckpointing, AwsCredentialsProvider dynamoDBCredentialsProvider,
String regionName, long shutdownGraceMillis, long schedulerInitializationBackoffTimeMillis) { AwsCredentialsProvider cloudWatchCredentialsProvider,
this(applicationName, streamName, kinesisEndpoint, null, initialPositionInStream, kinesisCredentialsProvider, long failoverTimeMillis,
dynamoDBCredentialsProvider, cloudWatchCredentialsProvider, failoverTimeMillis, workerId, maxRecords, String workerId,
idleTimeBetweenReadsInMillis, callProcessRecordsEvenForEmptyRecordList, parentShardPollIntervalMillis, int maxRecords,
shardSyncIntervalMillis, cleanupTerminatedShardsBeforeExpiry, taskBackoffTimeMillis, long idleTimeBetweenReadsInMillis,
metricsBufferTimeMillis, metricsMaxQueueSize, validateSequenceNumberBeforeCheckpointing, regionName, boolean callProcessRecordsEvenForEmptyRecordList,
shutdownGraceMillis, schedulerInitializationBackoffTimeMillis); long parentShardPollIntervalMillis,
long shardSyncIntervalMillis,
boolean cleanupTerminatedShardsBeforeExpiry,
long taskBackoffTimeMillis,
long metricsBufferTimeMillis,
int metricsMaxQueueSize,
boolean validateSequenceNumberBeforeCheckpointing,
String regionName,
long shutdownGraceMillis,
long schedulerInitializationBackoffTimeMillis) {
this(
applicationName,
streamName,
kinesisEndpoint,
null,
initialPositionInStream,
kinesisCredentialsProvider,
dynamoDBCredentialsProvider,
cloudWatchCredentialsProvider,
failoverTimeMillis,
workerId,
maxRecords,
idleTimeBetweenReadsInMillis,
callProcessRecordsEvenForEmptyRecordList,
parentShardPollIntervalMillis,
shardSyncIntervalMillis,
cleanupTerminatedShardsBeforeExpiry,
taskBackoffTimeMillis,
metricsBufferTimeMillis,
metricsMaxQueueSize,
validateSequenceNumberBeforeCheckpointing,
regionName,
shutdownGraceMillis,
schedulerInitializationBackoffTimeMillis);
} }
/** /**
@ -449,15 +502,30 @@ public class KinesisClientLibConfiguration {
*/ */
// CHECKSTYLE:IGNORE HiddenFieldCheck FOR NEXT 26 LINES // CHECKSTYLE:IGNORE HiddenFieldCheck FOR NEXT 26 LINES
// CHECKSTYLE:IGNORE ParameterNumber FOR NEXT 26 LINES // CHECKSTYLE:IGNORE ParameterNumber FOR NEXT 26 LINES
public KinesisClientLibConfiguration(String applicationName, String streamName, String kinesisEndpoint, public KinesisClientLibConfiguration(
String dynamoDBEndpoint, InitialPositionInStream initialPositionInStream, String applicationName,
AwsCredentialsProvider kinesisCredentialsProvider, AwsCredentialsProvider dynamoDBCredentialsProvider, String streamName,
AwsCredentialsProvider cloudWatchCredentialsProvider, long failoverTimeMillis, String workerId, String kinesisEndpoint,
int maxRecords, long idleTimeBetweenReadsInMillis, boolean callProcessRecordsEvenForEmptyRecordList, String dynamoDBEndpoint,
long parentShardPollIntervalMillis, long shardSyncIntervalMillis, InitialPositionInStream initialPositionInStream,
boolean cleanupTerminatedShardsBeforeExpiry, long taskBackoffTimeMillis, long metricsBufferTimeMillis, AwsCredentialsProvider kinesisCredentialsProvider,
int metricsMaxQueueSize, boolean validateSequenceNumberBeforeCheckpointing, String regionName, AwsCredentialsProvider dynamoDBCredentialsProvider,
long shutdownGraceMillis, long schedulerInitializationBackoffTimeMillis) { AwsCredentialsProvider cloudWatchCredentialsProvider,
long failoverTimeMillis,
String workerId,
int maxRecords,
long idleTimeBetweenReadsInMillis,
boolean callProcessRecordsEvenForEmptyRecordList,
long parentShardPollIntervalMillis,
long shardSyncIntervalMillis,
boolean cleanupTerminatedShardsBeforeExpiry,
long taskBackoffTimeMillis,
long metricsBufferTimeMillis,
int metricsMaxQueueSize,
boolean validateSequenceNumberBeforeCheckpointing,
String regionName,
long shutdownGraceMillis,
long schedulerInitializationBackoffTimeMillis) {
// Check following values are greater than zero // Check following values are greater than zero
checkIsValuePositive("FailoverTimeMillis", failoverTimeMillis); checkIsValuePositive("FailoverTimeMillis", failoverTimeMillis);
checkIsValuePositive("IdleTimeBetweenReadsInMillis", idleTimeBetweenReadsInMillis); checkIsValuePositive("IdleTimeBetweenReadsInMillis", idleTimeBetweenReadsInMillis);
@ -495,8 +563,8 @@ public class KinesisClientLibConfiguration {
this.maxLeasesToStealAtOneTime = DEFAULT_MAX_LEASES_TO_STEAL_AT_ONE_TIME; this.maxLeasesToStealAtOneTime = DEFAULT_MAX_LEASES_TO_STEAL_AT_ONE_TIME;
this.initialLeaseTableReadCapacity = DEFAULT_INITIAL_LEASE_TABLE_READ_CAPACITY; this.initialLeaseTableReadCapacity = DEFAULT_INITIAL_LEASE_TABLE_READ_CAPACITY;
this.initialLeaseTableWriteCapacity = DEFAULT_INITIAL_LEASE_TABLE_WRITE_CAPACITY; this.initialLeaseTableWriteCapacity = DEFAULT_INITIAL_LEASE_TABLE_WRITE_CAPACITY;
this.initialPositionInStreamExtended = InitialPositionInStreamExtended this.initialPositionInStreamExtended =
.newInitialPosition(initialPositionInStream); InitialPositionInStreamExtended.newInitialPosition(initialPositionInStream);
this.skipShardSyncAtWorkerInitializationIfLeasesExist = DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST; this.skipShardSyncAtWorkerInitializationIfLeasesExist = DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST;
this.shardPrioritization = DEFAULT_SHARD_PRIORITIZATION; this.shardPrioritization = DEFAULT_SHARD_PRIORITIZATION;
this.recordsFetcherFactory = new SimpleRecordsFetcherFactory(); this.recordsFetcherFactory = new SimpleRecordsFetcherFactory();
@ -559,15 +627,30 @@ public class KinesisClientLibConfiguration {
*/ */
// CHECKSTYLE:IGNORE HiddenFieldCheck FOR NEXT 26 LINES // CHECKSTYLE:IGNORE HiddenFieldCheck FOR NEXT 26 LINES
// CHECKSTYLE:IGNORE ParameterNumber FOR NEXT 26 LINES // CHECKSTYLE:IGNORE ParameterNumber FOR NEXT 26 LINES
public KinesisClientLibConfiguration(String applicationName, String streamName, String kinesisEndpoint, public KinesisClientLibConfiguration(
String dynamoDBEndpoint, InitialPositionInStream initialPositionInStream, String applicationName,
AwsCredentialsProvider kinesisCredentialsProvider, AwsCredentialsProvider dynamoDBCredentialsProvider, String streamName,
AwsCredentialsProvider cloudWatchCredentialsProvider, long failoverTimeMillis, String workerId, String kinesisEndpoint,
int maxRecords, long idleTimeBetweenReadsInMillis, boolean callProcessRecordsEvenForEmptyRecordList, String dynamoDBEndpoint,
long parentShardPollIntervalMillis, long shardSyncIntervalMillis, InitialPositionInStream initialPositionInStream,
boolean cleanupTerminatedShardsBeforeExpiry, long taskBackoffTimeMillis, long metricsBufferTimeMillis, AwsCredentialsProvider kinesisCredentialsProvider,
int metricsMaxQueueSize, boolean validateSequenceNumberBeforeCheckpointing, String regionName, AwsCredentialsProvider dynamoDBCredentialsProvider,
RecordsFetcherFactory recordsFetcherFactory, long schedulerInitializationBackoffTimeMillis) { AwsCredentialsProvider cloudWatchCredentialsProvider,
long failoverTimeMillis,
String workerId,
int maxRecords,
long idleTimeBetweenReadsInMillis,
boolean callProcessRecordsEvenForEmptyRecordList,
long parentShardPollIntervalMillis,
long shardSyncIntervalMillis,
boolean cleanupTerminatedShardsBeforeExpiry,
long taskBackoffTimeMillis,
long metricsBufferTimeMillis,
int metricsMaxQueueSize,
boolean validateSequenceNumberBeforeCheckpointing,
String regionName,
RecordsFetcherFactory recordsFetcherFactory,
long schedulerInitializationBackoffTimeMillis) {
// Check following values are greater than zero // Check following values are greater than zero
checkIsValuePositive("FailoverTimeMillis", failoverTimeMillis); checkIsValuePositive("FailoverTimeMillis", failoverTimeMillis);
checkIsValuePositive("IdleTimeBetweenReadsInMillis", idleTimeBetweenReadsInMillis); checkIsValuePositive("IdleTimeBetweenReadsInMillis", idleTimeBetweenReadsInMillis);
@ -607,8 +690,8 @@ public class KinesisClientLibConfiguration {
this.maxLeasesToStealAtOneTime = DEFAULT_MAX_LEASES_TO_STEAL_AT_ONE_TIME; this.maxLeasesToStealAtOneTime = DEFAULT_MAX_LEASES_TO_STEAL_AT_ONE_TIME;
this.initialLeaseTableReadCapacity = DEFAULT_INITIAL_LEASE_TABLE_READ_CAPACITY; this.initialLeaseTableReadCapacity = DEFAULT_INITIAL_LEASE_TABLE_READ_CAPACITY;
this.initialLeaseTableWriteCapacity = DEFAULT_INITIAL_LEASE_TABLE_WRITE_CAPACITY; this.initialLeaseTableWriteCapacity = DEFAULT_INITIAL_LEASE_TABLE_WRITE_CAPACITY;
this.initialPositionInStreamExtended = InitialPositionInStreamExtended this.initialPositionInStreamExtended =
.newInitialPosition(initialPositionInStream); InitialPositionInStreamExtended.newInitialPosition(initialPositionInStream);
this.skipShardSyncAtWorkerInitializationIfLeasesExist = DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST; this.skipShardSyncAtWorkerInitializationIfLeasesExist = DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST;
this.shardPrioritization = DEFAULT_SHARD_PRIORITIZATION; this.shardPrioritization = DEFAULT_SHARD_PRIORITIZATION;
this.recordsFetcherFactory = recordsFetcherFactory; this.recordsFetcherFactory = recordsFetcherFactory;
@ -933,8 +1016,8 @@ public class KinesisClientLibConfiguration {
*/ */
public KinesisClientLibConfiguration withInitialPositionInStream(InitialPositionInStream initialPositionInStream) { public KinesisClientLibConfiguration withInitialPositionInStream(InitialPositionInStream initialPositionInStream) {
this.initialPositionInStream = initialPositionInStream; this.initialPositionInStream = initialPositionInStream;
this.initialPositionInStreamExtended = InitialPositionInStreamExtended this.initialPositionInStreamExtended =
.newInitialPosition(initialPositionInStream); InitialPositionInStreamExtended.newInitialPosition(initialPositionInStream);
return this; return this;
} }
@ -984,8 +1067,8 @@ public class KinesisClientLibConfiguration {
public KinesisClientLibConfiguration withMaxRecords(int maxRecords) { public KinesisClientLibConfiguration withMaxRecords(int maxRecords) {
checkIsValuePositive("MaxRecords", (long) maxRecords); checkIsValuePositive("MaxRecords", (long) maxRecords);
if (maxRecords > DEFAULT_MAX_RECORDS) { if (maxRecords > DEFAULT_MAX_RECORDS) {
throw new IllegalArgumentException( throw new IllegalArgumentException("maxRecords must be less than or equal to " + DEFAULT_MAX_RECORDS
"maxRecords must be less than or equal to " + DEFAULT_MAX_RECORDS + " but current value is " + maxRecords); + " but current value is " + maxRecords);
} }
this.maxRecords = maxRecords; this.maxRecords = maxRecords;
return this; return this;
@ -1145,8 +1228,10 @@ public class KinesisClientLibConfiguration {
} else if (metricsEnabledDimensions.contains(MetricsScope.METRICS_DIMENSIONS_ALL)) { } else if (metricsEnabledDimensions.contains(MetricsScope.METRICS_DIMENSIONS_ALL)) {
this.metricsEnabledDimensions = METRICS_DIMENSIONS_ALL; this.metricsEnabledDimensions = METRICS_DIMENSIONS_ALL;
} else { } else {
this.metricsEnabledDimensions = ImmutableSet.<String> builder().addAll(metricsEnabledDimensions) this.metricsEnabledDimensions = ImmutableSet.<String>builder()
.addAll(METRICS_ALWAYS_ENABLED_DIMENSIONS).build(); .addAll(metricsEnabledDimensions)
.addAll(METRICS_ALWAYS_ENABLED_DIMENSIONS)
.build();
} }
return this; return this;
} }
@ -1277,7 +1362,8 @@ public class KinesisClientLibConfiguration {
* @return this configuration object * @return this configuration object
*/ */
public KinesisClientLibConfiguration withMaxLeaseRenewalThreads(int maxLeaseRenewalThreads) { public KinesisClientLibConfiguration withMaxLeaseRenewalThreads(int maxLeaseRenewalThreads) {
Validate.isTrue(maxLeaseRenewalThreads > 2, Validate.isTrue(
maxLeaseRenewalThreads > 2,
"The maximum number of lease renewal threads must be greater than or equal to 2."); "The maximum number of lease renewal threads must be greater than or equal to 2.");
this.maxLeaseRenewalThreads = maxLeaseRenewalThreads; this.maxLeaseRenewalThreads = maxLeaseRenewalThreads;
@ -1337,7 +1423,8 @@ public class KinesisClientLibConfiguration {
* @return KinesisClientLibConfiguration * @return KinesisClientLibConfiguration
*/ */
public KinesisClientLibConfiguration withDataFetchingStrategy(String dataFetchingStrategy) { public KinesisClientLibConfiguration withDataFetchingStrategy(String dataFetchingStrategy) {
this.recordsFetcherFactory.dataFetchingStrategy(DataFetchingStrategy.valueOf(dataFetchingStrategy.toUpperCase())); this.recordsFetcherFactory.dataFetchingStrategy(
DataFetchingStrategy.valueOf(dataFetchingStrategy.toUpperCase()));
return this; return this;
} }
@ -1423,7 +1510,8 @@ public class KinesisClientLibConfiguration {
* Interval in milliseconds between retrying the scheduler initialization. * Interval in milliseconds between retrying the scheduler initialization.
* @return * @return
*/ */
public KinesisClientLibConfiguration withSchedulerInitializationBackoffTimeMillis(long schedulerInitializationBackoffTimeMillis) { public KinesisClientLibConfiguration withSchedulerInitializationBackoffTimeMillis(
long schedulerInitializationBackoffTimeMillis) {
checkIsValuePositive("schedulerInitializationBackoffTimeMillis", schedulerInitializationBackoffTimeMillis); checkIsValuePositive("schedulerInitializationBackoffTimeMillis", schedulerInitializationBackoffTimeMillis);
this.schedulerInitializationBackoffTimeMillis = schedulerInitializationBackoffTimeMillis; this.schedulerInitializationBackoffTimeMillis = schedulerInitializationBackoffTimeMillis;
return this; return this;

View file

@ -23,8 +23,7 @@ import lombok.extern.slf4j.Slf4j;
*/ */
@Slf4j @Slf4j
class DrainChildSTDERRTask extends LineReaderTask<Boolean> { class DrainChildSTDERRTask extends LineReaderTask<Boolean> {
DrainChildSTDERRTask() { DrainChildSTDERRTask() {}
}
@Override @Override
protected HandleLineResult<Boolean> handleLine(String line) { protected HandleLineResult<Boolean> handleLine(String line) {

View file

@ -22,23 +22,22 @@ import lombok.extern.slf4j.Slf4j;
* This class is used to drain the STDOUT of the child process. After the child process has been given a shutdown * This class is used to drain the STDOUT of the child process. After the child process has been given a shutdown
* message and responded indicating that it is shutdown, we attempt to close the input and outputs of that process so * message and responded indicating that it is shutdown, we attempt to close the input and outputs of that process so
* that the process can exit. * that the process can exit.
* *
* To understand why this is necessary, consider the following scenario: * To understand why this is necessary, consider the following scenario:
* *
* <ol> * <ol>
* <li>Child process responds that it is done with shutdown.</li> * <li>Child process responds that it is done with shutdown.</li>
* <li>Child process prints debugging text to STDOUT that fills the pipe buffer so child becomes blocked.</li> * <li>Child process prints debugging text to STDOUT that fills the pipe buffer so child becomes blocked.</li>
* <li>Parent process doesn't drain child process's STDOUT.</li> * <li>Parent process doesn't drain child process's STDOUT.</li>
* <li>Child process remains blocked.</li> * <li>Child process remains blocked.</li>
* </ol> * </ol>
* *
* To prevent the child process from becoming blocked in this way, it is the responsibility of the parent process to * To prevent the child process from becoming blocked in this way, it is the responsibility of the parent process to
* drain the child process's STDOUT. We reprint each drained line to our log to permit debugging. * drain the child process's STDOUT. We reprint each drained line to our log to permit debugging.
*/ */
@Slf4j @Slf4j
class DrainChildSTDOUTTask extends LineReaderTask<Boolean> { class DrainChildSTDOUTTask extends LineReaderTask<Boolean> {
DrainChildSTDOUTTask() { DrainChildSTDOUTTask() {}
}
@Override @Override
protected HandleLineResult<Boolean> handleLine(String line) { protected HandleLineResult<Boolean> handleLine(String line) {

View file

@ -17,10 +17,9 @@ package software.amazon.kinesis.multilang;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.IOException; import java.io.IOException;
import software.amazon.kinesis.multilang.messages.Message;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import software.amazon.kinesis.multilang.messages.Message;
/** /**
* Gets the next message off the STDOUT of the child process. Throws an exception if a message is not found before the * Gets the next message off the STDOUT of the child process. Throws an exception if a message is not found before the
@ -34,7 +33,7 @@ class GetNextMessageTask extends LineReaderTask<Message> {
/** /**
* Constructor. * Constructor.
* *
* @param objectMapper An object mapper for decoding json messages from the input stream. * @param objectMapper An object mapper for decoding json messages from the input stream.
*/ */
GetNextMessageTask(ObjectMapper objectMapper) { GetNextMessageTask(ObjectMapper objectMapper) {
@ -43,7 +42,7 @@ class GetNextMessageTask extends LineReaderTask<Message> {
/** /**
* Checks if a line is an empty line. * Checks if a line is an empty line.
* *
* @param line A string * @param line A string
* @return True if the line is an empty string, i.e. "", false otherwise. * @return True if the line is an empty string, i.e. "", false otherwise.
*/ */
@ -71,8 +70,10 @@ class GetNextMessageTask extends LineReaderTask<Message> {
@Override @Override
protected Message returnAfterException(Exception e) { protected Message returnAfterException(Exception e) {
throw new RuntimeException("Encountered an error while reading a line from STDIN for shard " + getShardId() throw new RuntimeException(
+ " so won't be able to return a message.", e); "Encountered an error while reading a line from STDIN for shard " + getShardId()
+ " so won't be able to return a message.",
e);
} }
@Override @Override

View file

@ -30,7 +30,7 @@ import lombok.extern.slf4j.Slf4j;
* <li> {@link #returnAfterEndOfInput()}</li> * <li> {@link #returnAfterEndOfInput()}</li>
* <li> {@link #returnAfterException(Exception)}</li> * <li> {@link #returnAfterException(Exception)}</li>
* </ol> * </ol>
* *
* @param <T> * @param <T>
*/ */
@Slf4j @Slf4j
@ -41,8 +41,7 @@ abstract class LineReaderTask<T> implements Callable<T> {
private String shardId; private String shardId;
LineReaderTask() { LineReaderTask() {}
}
/** /**
* Reads lines off the input stream until a return value is set, or an exception is encountered, or the end of the * Reads lines off the input stream until a return value is set, or an exception is encountered, or the end of the
@ -72,7 +71,7 @@ abstract class LineReaderTask<T> implements Callable<T> {
* return from the {@link #call()} function by having a value, indicating that value should be returned immediately * return from the {@link #call()} function by having a value, indicating that value should be returned immediately
* without reading further, or not having a value, indicating that more lines of input need to be read before * without reading further, or not having a value, indicating that more lines of input need to be read before
* returning. * returning.
* *
* @param line A line read from the input stream. * @param line A line read from the input stream.
* @return HandleLineResult<T> which may or may not have a has return value, indicating to return or not return yet * @return HandleLineResult<T> which may or may not have a has return value, indicating to return or not return yet
* respectively. * respectively.
@ -83,7 +82,7 @@ abstract class LineReaderTask<T> implements Callable<T> {
* This method will be called if there is an error while reading from the input stream. The return value of this * This method will be called if there is an error while reading from the input stream. The return value of this
* method will be returned as the result of this Callable unless an Exception is thrown. If an Exception is thrown * method will be returned as the result of this Callable unless an Exception is thrown. If an Exception is thrown
* then that exception will be thrown by the Callable. * then that exception will be thrown by the Callable.
* *
* @param e An exception that occurred while reading from the input stream. * @param e An exception that occurred while reading from the input stream.
* @return What to return. * @return What to return.
*/ */
@ -93,7 +92,7 @@ abstract class LineReaderTask<T> implements Callable<T> {
* This method will be called once the end of the input stream is reached. The return value of this method will be * This method will be called once the end of the input stream is reached. The return value of this method will be
* returned as the result of this Callable. Implementations of this method are welcome to throw a runtime exception * returned as the result of this Callable. Implementations of this method are welcome to throw a runtime exception
* to indicate that the task was unsuccessful. * to indicate that the task was unsuccessful.
* *
* @return What to return. * @return What to return.
*/ */
protected abstract T returnAfterEndOfInput(); protected abstract T returnAfterEndOfInput();
@ -101,7 +100,7 @@ abstract class LineReaderTask<T> implements Callable<T> {
/** /**
* Allows subclasses to provide more detailed logs. Specifically, this allows the drain tasks and GetNextMessageTask * Allows subclasses to provide more detailed logs. Specifically, this allows the drain tasks and GetNextMessageTask
* to log which shard they're working on. * to log which shard they're working on.
* *
* @return The shard id * @return The shard id
*/ */
public String getShardId() { public String getShardId() {
@ -110,7 +109,7 @@ abstract class LineReaderTask<T> implements Callable<T> {
/** /**
* The description should be a string explaining what this particular LineReader class does. * The description should be a string explaining what this particular LineReader class does.
* *
* @return The description. * @return The description.
*/ */
public String getDescription() { public String getDescription() {
@ -121,7 +120,7 @@ abstract class LineReaderTask<T> implements Callable<T> {
* The result of a call to {@link LineReaderTask#handleLine(String)}. Allows implementations of that method to * The result of a call to {@link LineReaderTask#handleLine(String)}. Allows implementations of that method to
* indicate whether a particular invocation of that method produced a return for this task or not. If a return value * indicate whether a particular invocation of that method produced a return for this task or not. If a return value
* doesn't exist the {@link #call()} method will continue to the next line. * doesn't exist the {@link #call()} method will continue to the next line.
* *
* @param <V> * @param <V>
*/ */
protected class HandleLineResult<V> { protected class HandleLineResult<V> {
@ -158,7 +157,7 @@ abstract class LineReaderTask<T> implements Callable<T> {
* {@link MultiLangShardRecordProcessor#initialize(String)} is called. So we follow a pattern where the attributes are * {@link MultiLangShardRecordProcessor#initialize(String)} is called. So we follow a pattern where the attributes are
* set inside this method instead of the constructor so that this object will be initialized when all its attributes * set inside this method instead of the constructor so that this object will be initialized when all its attributes
* are known to the record processor. * are known to the record processor.
* *
* @param stream * @param stream
* @param shardId * @param shardId
* @param description * @param description
@ -180,5 +179,4 @@ abstract class LineReaderTask<T> implements Callable<T> {
this.description = description; this.description = description;
return this; return this;
} }
} }

View file

@ -20,19 +20,19 @@ import java.io.InputStreamReader;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future; import java.util.concurrent.Future;
import software.amazon.kinesis.multilang.messages.Message;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import software.amazon.kinesis.multilang.messages.Message;
/** /**
* Provides methods for interacting with the child process's STDOUT. * Provides methods for interacting with the child process's STDOUT.
* *
* {@link #getNextMessageFromSTDOUT()} reads lines from the child process's STDOUT and attempts to decode a * {@link #getNextMessageFromSTDOUT()} reads lines from the child process's STDOUT and attempts to decode a
* {@link Message} object from each line. A child process's STDOUT could have lines that don't contain data related to * {@link Message} object from each line. A child process's STDOUT could have lines that don't contain data related to
* the multi-language protocol, such as when the child process prints debugging information to its STDOUT (instead of * the multi-language protocol, such as when the child process prints debugging information to its STDOUT (instead of
* logging to a file), also when a child processes writes a Message it is expected to prepend and append a new line * logging to a file), also when a child processes writes a Message it is expected to prepend and append a new line
* character to their message to help ensure that it is isolated on a line all by itself which results in empty lines * character to their message to help ensure that it is isolated on a line all by itself which results in empty lines
* being present in STDOUT. Lines which cannot be decoded to a Message object are ignored. * being present in STDOUT. Lines which cannot be decoded to a Message object are ignored.
* *
* {@link #drainSTDOUT()} simply reads all data from the child process's STDOUT until the stream is closed. * {@link #drainSTDOUT()} simply reads all data from the child process's STDOUT until the stream is closed.
*/ */
class MessageReader { class MessageReader {
@ -48,19 +48,18 @@ class MessageReader {
/** /**
* Use the initialize methods after construction. * Use the initialize methods after construction.
*/ */
MessageReader() { MessageReader() {}
}
/** /**
* Returns a future which represents an attempt to read the next message in the child process's STDOUT. If the task * Returns a future which represents an attempt to read the next message in the child process's STDOUT. If the task
* is successful, the result of the future will be the next message found in the child process's STDOUT, if the task * is successful, the result of the future will be the next message found in the child process's STDOUT, if the task
* is unable to find a message before the child process's STDOUT is closed, or reading from STDOUT causes an * is unable to find a message before the child process's STDOUT is closed, or reading from STDOUT causes an
* IOException, then an execution exception will be generated by this future. * IOException, then an execution exception will be generated by this future.
* *
* The task employed by this method reads from the child process's STDOUT line by line. The task attempts to decode * The task employed by this method reads from the child process's STDOUT line by line. The task attempts to decode
* each line into a {@link Message} object. Lines that fail to decode to a Message are ignored and the task * each line into a {@link Message} object. Lines that fail to decode to a Message are ignored and the task
* continues to the next line until it finds a Message. * continues to the next line until it finds a Message.
* *
* @return * @return
*/ */
Future<Message> getNextMessageFromSTDOUT() { Future<Message> getNextMessageFromSTDOUT() {
@ -73,7 +72,7 @@ class MessageReader {
* Returns a future that represents a computation that drains the STDOUT of the child process. That future's result * Returns a future that represents a computation that drains the STDOUT of the child process. That future's result
* is true if the end of the child's STDOUT is reached, its result is false if there was an error while reading from * is true if the end of the child's STDOUT is reached, its result is false if there was an error while reading from
* the stream. This task will log all the lines it drains to permit debugging. * the stream. This task will log all the lines it drains to permit debugging.
* *
* @return * @return
*/ */
Future<Boolean> drainSTDOUT() { Future<Boolean> drainSTDOUT() {
@ -89,19 +88,16 @@ class MessageReader {
* {@link MultiLangShardRecordProcessor#initialize(String)} is called. So we follow a pattern where the attributes are * {@link MultiLangShardRecordProcessor#initialize(String)} is called. So we follow a pattern where the attributes are
* set inside this method instead of the constructor so that this object will be initialized when all its attributes * set inside this method instead of the constructor so that this object will be initialized when all its attributes
* are known to the record processor. * are known to the record processor.
* *
* @param stream Used to read messages from the subprocess. * @param stream Used to read messages from the subprocess.
* @param shardId The shard we're working on. * @param shardId The shard we're working on.
* @param objectMapper The object mapper to decode messages. * @param objectMapper The object mapper to decode messages.
* @param executorService An executor service to run tasks in. * @param executorService An executor service to run tasks in.
*/ */
MessageReader initialize(InputStream stream, MessageReader initialize(
String shardId, InputStream stream, String shardId, ObjectMapper objectMapper, ExecutorService executorService) {
ObjectMapper objectMapper, return this.initialize(
ExecutorService executorService) { new BufferedReader(new InputStreamReader(stream)), shardId, objectMapper, executorService);
return this.initialize(new BufferedReader(new InputStreamReader(stream)), shardId, objectMapper,
executorService);
} }
/** /**
@ -110,10 +106,8 @@ class MessageReader {
* @param objectMapper The object mapper to decode messages. * @param objectMapper The object mapper to decode messages.
* @param executorService An executor service to run tasks in. * @param executorService An executor service to run tasks in.
*/ */
MessageReader initialize(BufferedReader reader, MessageReader initialize(
String shardId, BufferedReader reader, String shardId, ObjectMapper objectMapper, ExecutorService executorService) {
ObjectMapper objectMapper,
ExecutorService executorService) {
this.reader = reader; this.reader = reader;
this.shardId = shardId; this.shardId = shardId;
this.objectMapper = objectMapper; this.objectMapper = objectMapper;

View file

@ -23,7 +23,6 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future; import java.util.concurrent.Future;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import software.amazon.kinesis.lifecycle.events.InitializationInput; import software.amazon.kinesis.lifecycle.events.InitializationInput;
import software.amazon.kinesis.lifecycle.events.LeaseLostInput; import software.amazon.kinesis.lifecycle.events.LeaseLostInput;
@ -55,13 +54,12 @@ class MessageWriter {
/** /**
* Use initialize method after construction. * Use initialize method after construction.
*/ */
MessageWriter() { MessageWriter() {}
}
/** /**
* Writes the message then writes the line separator provided by the system. Flushes each message to guarantee it * Writes the message then writes the line separator provided by the system. Flushes each message to guarantee it
* is delivered as soon as possible to the subprocess. * is delivered as soon as possible to the subprocess.
* *
* @param message A message to be written to the subprocess. * @param message A message to be written to the subprocess.
* @return * @return
* @throws IOException * @throws IOException
@ -76,7 +74,10 @@ class MessageWriter {
*/ */
synchronized (writer) { synchronized (writer) {
writer.write(message, 0, message.length()); writer.write(message, 0, message.length());
writer.write(System.lineSeparator(), 0, System.lineSeparator().length()); writer.write(
System.lineSeparator(),
0,
System.lineSeparator().length());
writer.flush(); writer.flush();
} }
log.info("Message size == {} bytes for shard {}", message.getBytes().length, shardId); log.info("Message size == {} bytes for shard {}", message.getBytes().length, shardId);
@ -98,7 +99,7 @@ class MessageWriter {
/** /**
* Converts the message to a JSON string and writes it to the subprocess. * Converts the message to a JSON string and writes it to the subprocess.
* *
* @param message A message to be written to the subprocess. * @param message A message to be written to the subprocess.
* @return * @return
*/ */
@ -108,9 +109,9 @@ class MessageWriter {
String jsonText = objectMapper.writeValueAsString(message); String jsonText = objectMapper.writeValueAsString(message);
return writeMessageToOutput(jsonText); return writeMessageToOutput(jsonText);
} catch (IOException e) { } catch (IOException e) {
String errorMessage = String errorMessage = String.format(
String.format("Encountered I/O error while writing %s action to subprocess", message.getClass() "Encountered I/O error while writing %s action to subprocess",
.getSimpleName()); message.getClass().getSimpleName());
log.error(errorMessage, e); log.error(errorMessage, e);
throw new RuntimeException(errorMessage, e); throw new RuntimeException(errorMessage, e);
} }
@ -118,7 +119,7 @@ class MessageWriter {
/** /**
* Writes an {@link InitializeMessage} to the subprocess. * Writes an {@link InitializeMessage} to the subprocess.
* *
* @param initializationInput * @param initializationInput
* contains information about the shard being initialized * contains information about the shard being initialized
*/ */
@ -128,7 +129,7 @@ class MessageWriter {
/** /**
* Writes a {@link ProcessRecordsMessage} message to the subprocess. * Writes a {@link ProcessRecordsMessage} message to the subprocess.
* *
* @param processRecordsInput * @param processRecordsInput
* the records, and associated metadata to be processed. * the records, and associated metadata to be processed.
*/ */
@ -138,7 +139,7 @@ class MessageWriter {
/** /**
* Writes the lease lost message to the sub process. * Writes the lease lost message to the sub process.
* *
* @param leaseLostInput * @param leaseLostInput
* the lease lost input. This is currently unused as lease loss doesn't actually have anything in it * the lease lost input. This is currently unused as lease loss doesn't actually have anything in it
* @return A future that is set when the message has been written. * @return A future that is set when the message has been written.
@ -149,7 +150,7 @@ class MessageWriter {
/** /**
* Writes a message to the sub process indicating that the shard has ended * Writes a message to the sub process indicating that the shard has ended
* *
* @param shardEndedInput * @param shardEndedInput
* the shard end input. This is currently unused as the checkpoint is extracted, and used by the caller. * the shard end input. This is currently unused as the checkpoint is extracted, and used by the caller.
* @return A future that is set when the message has been written. * @return A future that is set when the message has been written.
@ -167,7 +168,7 @@ class MessageWriter {
/** /**
* Writes a {@link CheckpointMessage} to the subprocess. * Writes a {@link CheckpointMessage} to the subprocess.
* *
* @param sequenceNumber * @param sequenceNumber
* The sequence number that was checkpointed. * The sequence number that was checkpointed.
* @param subSequenceNumber * @param subSequenceNumber
@ -175,14 +176,14 @@ class MessageWriter {
* @param throwable * @param throwable
* The exception that was thrown by a checkpoint attempt. Null if one didn't occur. * The exception that was thrown by a checkpoint attempt. Null if one didn't occur.
*/ */
Future<Boolean> writeCheckpointMessageWithError(String sequenceNumber, Long subSequenceNumber, Future<Boolean> writeCheckpointMessageWithError(
Throwable throwable) { String sequenceNumber, Long subSequenceNumber, Throwable throwable) {
return writeMessage(new CheckpointMessage(sequenceNumber, subSequenceNumber, throwable)); return writeMessage(new CheckpointMessage(sequenceNumber, subSequenceNumber, throwable));
} }
/** /**
* Closes the output stream and prevents further attempts to write. * Closes the output stream and prevents further attempts to write.
* *
* @throws IOException Thrown when closing the writer fails * @throws IOException Thrown when closing the writer fails
*/ */
void close() throws IOException { void close() throws IOException {
@ -201,18 +202,16 @@ class MessageWriter {
* {@link MultiLangShardRecordProcessor (String)} is called. So we follow a pattern where the attributes are * {@link MultiLangShardRecordProcessor (String)} is called. So we follow a pattern where the attributes are
* set inside this method instead of the constructor so that this object will be initialized when all its attributes * set inside this method instead of the constructor so that this object will be initialized when all its attributes
* are known to the record processor. * are known to the record processor.
* *
* @param stream Used to write messages to the subprocess. * @param stream Used to write messages to the subprocess.
* @param shardId The shard we're working on. * @param shardId The shard we're working on.
* @param objectMapper The object mapper to encode messages. * @param objectMapper The object mapper to encode messages.
* @param executorService An executor service to run tasks in. * @param executorService An executor service to run tasks in.
*/ */
MessageWriter initialize(OutputStream stream, MessageWriter initialize(
String shardId, OutputStream stream, String shardId, ObjectMapper objectMapper, ExecutorService executorService) {
ObjectMapper objectMapper, return this.initialize(
ExecutorService executorService) { new BufferedWriter(new OutputStreamWriter(stream)), shardId, objectMapper, executorService);
return this.initialize(new BufferedWriter(new OutputStreamWriter(stream)), shardId, objectMapper,
executorService);
} }
/** /**
@ -221,15 +220,12 @@ class MessageWriter {
* @param objectMapper The object mapper to encode messages. * @param objectMapper The object mapper to encode messages.
* @param executorService An executor service to run tasks in. * @param executorService An executor service to run tasks in.
*/ */
MessageWriter initialize(BufferedWriter writer, MessageWriter initialize(
String shardId, BufferedWriter writer, String shardId, ObjectMapper objectMapper, ExecutorService executorService) {
ObjectMapper objectMapper,
ExecutorService executorService) {
this.writer = writer; this.writer = writer;
this.shardId = shardId; this.shardId = shardId;
this.objectMapper = objectMapper; this.objectMapper = objectMapper;
this.executorService = executorService; this.executorService = executorService;
return this; return this;
} }
} }

View file

@ -26,20 +26,18 @@ import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException; import java.util.concurrent.TimeoutException;
import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.joran.JoranConfigurator;
import ch.qos.logback.core.joran.spi.JoranException;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import lombok.Data;
import lombok.experimental.Accessors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.joran.JoranConfigurator;
import ch.qos.logback.core.joran.spi.JoranException;
import lombok.Data;
import lombok.experimental.Accessors;
import lombok.extern.slf4j.Slf4j;
import software.amazon.kinesis.coordinator.Scheduler; import software.amazon.kinesis.coordinator.Scheduler;
/** /**
@ -75,11 +73,14 @@ public class MultiLangDaemon {
@Parameter @Parameter
List<String> parameters = new ArrayList<>(); List<String> parameters = new ArrayList<>();
@Parameter(names = { "-p", "--properties-file" }, description = "Properties file to be used with the KCL") @Parameter(
names = {"-p", "--properties-file"},
description = "Properties file to be used with the KCL")
String propertiesFile; String propertiesFile;
@Parameter(names = { "-l", @Parameter(
"--log-configuration" }, description = "File location of logback.xml to be override the default") names = {"-l", "--log-configuration"},
description = "File location of logback.xml to be override the default")
String logConfiguration; String logConfiguration;
} }
@ -102,7 +103,8 @@ public class MultiLangDaemon {
} }
JCommander buildJCommanderAndParseArgs(final MultiLangDaemonArguments arguments, final String[] args) { JCommander buildJCommanderAndParseArgs(final MultiLangDaemonArguments arguments, final String[] args) {
JCommander jCommander = JCommander.newBuilder().programName("amazon-kinesis-client MultiLangDaemon") JCommander jCommander = JCommander.newBuilder()
.programName("amazon-kinesis-client MultiLangDaemon")
.addObject(arguments) .addObject(arguments)
.build(); .build();
jCommander.parse(args); jCommander.parse(args);
@ -128,8 +130,8 @@ public class MultiLangDaemon {
} }
} }
void configureLogging(final String logConfiguration, final LoggerContext loggerContext, void configureLogging(
final JoranConfigurator configurator) { final String logConfiguration, final LoggerContext loggerContext, final JoranConfigurator configurator) {
loggerContext.reset(); loggerContext.reset();
try (InputStream inputStream = FileUtils.openInputStream(new File(logConfiguration))) { try (InputStream inputStream = FileUtils.openInputStream(new File(logConfiguration))) {
configurator.setContext(loggerContext); configurator.setContext(loggerContext);
@ -146,9 +148,8 @@ public class MultiLangDaemon {
if (arguments.parameters.size() == 1) { if (arguments.parameters.size() == 1) {
propertiesFile = arguments.parameters.get(0); propertiesFile = arguments.parameters.get(0);
} else { } else {
throw new RuntimeException( throw new RuntimeException("Expected a single argument, but found multiple arguments. Arguments: "
"Expected a single argument, but found multiple arguments. Arguments: " + String.join(", ", arguments.parameters));
+ String.join(", ", arguments.parameters));
} }
} }

View file

@ -26,10 +26,9 @@ import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import software.amazon.kinesis.multilang.config.KinesisClientLibConfigurator;
import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.common.util.concurrent.ThreadFactoryBuilder;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import software.amazon.kinesis.multilang.config.KinesisClientLibConfigurator;
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration; import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
import software.amazon.kinesis.retrieval.RetrievalConfig; import software.amazon.kinesis.retrieval.RetrievalConfig;
@ -53,7 +52,7 @@ public class MultiLangDaemonConfig {
/** /**
* Constructor. * Constructor.
* *
* @param propertiesFile * @param propertiesFile
* The location of the properties file. * The location of the properties file.
* @throws IOException * @throws IOException
@ -66,7 +65,7 @@ public class MultiLangDaemonConfig {
} }
/** /**
* *
* @param propertiesFile * @param propertiesFile
* The location of the properties file. * The location of the properties file.
* @param classLoader * @param classLoader
@ -82,7 +81,7 @@ public class MultiLangDaemonConfig {
} }
/** /**
* *
* @param propertiesFile * @param propertiesFile
* The location of the properties file. * The location of the properties file.
* @param classLoader * @param classLoader
@ -94,8 +93,9 @@ public class MultiLangDaemonConfig {
* @throws IllegalArgumentException * @throws IllegalArgumentException
* Thrown when the contents of the properties file are not as expected. * Thrown when the contents of the properties file are not as expected.
*/ */
public MultiLangDaemonConfig(String propertiesFile, ClassLoader classLoader, public MultiLangDaemonConfig(
KinesisClientLibConfigurator configurator) throws IOException, IllegalArgumentException { String propertiesFile, ClassLoader classLoader, KinesisClientLibConfigurator configurator)
throws IOException, IllegalArgumentException {
Properties properties = loadProperties(classLoader, propertiesFile); Properties properties = loadProperties(classLoader, propertiesFile);
if (!validateProperties(properties)) { if (!validateProperties(properties)) {
throw new IllegalArgumentException( throw new IllegalArgumentException(
@ -107,11 +107,14 @@ public class MultiLangDaemonConfig {
multiLangDaemonConfiguration = configurator.getConfiguration(properties); multiLangDaemonConfiguration = configurator.getConfiguration(properties);
executorService = buildExecutorService(properties); executorService = buildExecutorService(properties);
recordProcessorFactory = new MultiLangRecordProcessorFactory(executableName, executorService, recordProcessorFactory =
multiLangDaemonConfiguration); new MultiLangRecordProcessorFactory(executableName, executorService, multiLangDaemonConfiguration);
log.info("Running {} to process stream {} with executable {}", multiLangDaemonConfiguration.getApplicationName(), log.info(
multiLangDaemonConfiguration.getStreamName(), executableName); "Running {} to process stream {} with executable {}",
multiLangDaemonConfiguration.getApplicationName(),
multiLangDaemonConfiguration.getStreamName(),
executableName);
prepare(processingLanguage); prepare(processingLanguage);
} }
@ -138,7 +141,7 @@ public class MultiLangDaemonConfig {
} }
log.info("MultiLangDaemon is adding the following fields to the User Agent: {}", userAgent.toString()); log.info("MultiLangDaemon is adding the following fields to the User Agent: {}", userAgent.toString());
// multiLangDaemonConfiguration.withUserAgent(userAgent.toString()); // multiLangDaemonConfiguration.withUserAgent(userAgent.toString());
} }
private static Properties loadProperties(ClassLoader classLoader, String propertiesFileName) throws IOException { private static Properties loadProperties(ClassLoader classLoader, String propertiesFileName) throws IOException {
@ -181,17 +184,22 @@ public class MultiLangDaemonConfig {
log.debug("Value for {} property is {}", PROP_MAX_ACTIVE_THREADS, maxActiveThreads); log.debug("Value for {} property is {}", PROP_MAX_ACTIVE_THREADS, maxActiveThreads);
if (maxActiveThreads <= 0) { if (maxActiveThreads <= 0) {
log.info("Using a cached thread pool."); log.info("Using a cached thread pool.");
return new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, new SynchronousQueue<>(), return new ThreadPoolExecutor(
builder.build()); 0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, new SynchronousQueue<>(), builder.build());
} else { } else {
log.info("Using a fixed thread pool with {} max active threads.", maxActiveThreads); log.info("Using a fixed thread pool with {} max active threads.", maxActiveThreads);
return new ThreadPoolExecutor(maxActiveThreads, maxActiveThreads, 0L, TimeUnit.MILLISECONDS, return new ThreadPoolExecutor(
new LinkedBlockingQueue<>(), builder.build()); maxActiveThreads,
maxActiveThreads,
0L,
TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<>(),
builder.build());
} }
} }
/** /**
* *
* @return A KinesisClientLibConfiguration object based on the properties file provided. * @return A KinesisClientLibConfiguration object based on the properties file provided.
*/ */
public MultiLangDaemonConfiguration getMultiLangDaemonConfiguration() { public MultiLangDaemonConfiguration getMultiLangDaemonConfiguration() {
@ -199,7 +207,7 @@ public class MultiLangDaemonConfig {
} }
/** /**
* *
* @return An executor service based on the properties file provided. * @return An executor service based on the properties file provided.
*/ */
public ExecutorService getExecutorService() { public ExecutorService getExecutorService() {
@ -207,7 +215,7 @@ public class MultiLangDaemonConfig {
} }
/** /**
* *
* @return A MultiLangRecordProcessorFactory based on the properties file provided. * @return A MultiLangRecordProcessorFactory based on the properties file provided.
*/ */
public MultiLangRecordProcessorFactory getRecordProcessorFactory() { public MultiLangRecordProcessorFactory getRecordProcessorFactory() {

View file

@ -61,8 +61,11 @@ class MultiLangProtocol {
* @param initializationInput * @param initializationInput
* information about the shard this processor is starting to process * information about the shard this processor is starting to process
*/ */
MultiLangProtocol(MessageReader messageReader, MessageWriter messageWriter, MultiLangProtocol(
InitializationInput initializationInput, MultiLangDaemonConfiguration configuration) { MessageReader messageReader,
MessageWriter messageWriter,
InitializationInput initializationInput,
MultiLangDaemonConfiguration configuration) {
this.messageReader = messageReader; this.messageReader = messageReader;
this.messageWriter = messageWriter; this.messageWriter = messageWriter;
this.initializationInput = initializationInput; this.initializationInput = initializationInput;
@ -82,7 +85,6 @@ class MultiLangProtocol {
*/ */
Future<Boolean> writeFuture = messageWriter.writeInitializeMessage(initializationInput); Future<Boolean> writeFuture = messageWriter.writeInitializeMessage(initializationInput);
return waitForStatusMessage(InitializeMessage.ACTION, null, writeFuture); return waitForStatusMessage(InitializeMessage.ACTION, null, writeFuture);
} }
/** /**
@ -100,7 +102,7 @@ class MultiLangProtocol {
/** /**
* Notifies the client process that the lease has been lost, and it needs to shutdown. * Notifies the client process that the lease has been lost, and it needs to shutdown.
* *
* @param leaseLostInput * @param leaseLostInput
* the lease lost input that is passed to the {@link MessageWriter} * the lease lost input that is passed to the {@link MessageWriter}
* @return true if the message was successfully writtem * @return true if the message was successfully writtem
@ -115,7 +117,9 @@ class MultiLangProtocol {
* @return * @return
*/ */
boolean shardEnded(ShardEndedInput shardEndedInput) { boolean shardEnded(ShardEndedInput shardEndedInput) {
return waitForStatusMessage(ShardEndedMessage.ACTION, shardEndedInput.checkpointer(), return waitForStatusMessage(
ShardEndedMessage.ACTION,
shardEndedInput.checkpointer(),
messageWriter.writeShardEndedMessage(shardEndedInput)); messageWriter.writeShardEndedMessage(shardEndedInput));
} }
@ -147,8 +151,8 @@ class MultiLangProtocol {
* The writing task. * The writing task.
* @return Whether or not this operation succeeded. * @return Whether or not this operation succeeded.
*/ */
private boolean waitForStatusMessage(String action, RecordProcessorCheckpointer checkpointer, private boolean waitForStatusMessage(
Future<Boolean> writeFuture) { String action, RecordProcessorCheckpointer checkpointer, Future<Boolean> writeFuture) {
boolean statusWasCorrect = waitForStatusMessage(action, checkpointer); boolean statusWasCorrect = waitForStatusMessage(action, checkpointer);
// Examine whether or not we failed somewhere along the line. // Examine whether or not we failed somewhere along the line.
@ -194,7 +198,7 @@ class MultiLangProtocol {
return false; return false;
} }
statusMessage = message.filter(m -> m instanceof StatusMessage).map(m -> (StatusMessage) m ); statusMessage = message.filter(m -> m instanceof StatusMessage).map(m -> (StatusMessage) m);
} }
return this.validateStatusMessage(statusMessage.get(), action); return this.validateStatusMessage(statusMessage.get(), action);
} }
@ -207,13 +211,17 @@ class MultiLangProtocol {
try { try {
return Optional.of(fm.get()); return Optional.of(fm.get());
} catch (InterruptedException e) { } catch (InterruptedException e) {
log.error("Interrupted while waiting for {} message for shard {}", action, log.error(
initializationInput.shardId(), e); "Interrupted while waiting for {} message for shard {}", action, initializationInput.shardId(), e);
} catch (ExecutionException e) { } catch (ExecutionException e) {
log.error("Failed to get status message for {} action for shard {}", action, log.error(
initializationInput.shardId(), e); "Failed to get status message for {} action for shard {}",
action,
initializationInput.shardId(),
e);
} catch (TimeoutException e) { } catch (TimeoutException e) {
log.error("Timedout to get status message for {} action for shard {}. Terminating...", log.error(
"Timedout to get status message for {} action for shard {}. Terminating...",
action, action,
initializationInput.shardId(), initializationInput.shardId(),
e); e);
@ -240,11 +248,14 @@ class MultiLangProtocol {
* @return Whether or not this operation succeeded. * @return Whether or not this operation succeeded.
*/ */
private boolean validateStatusMessage(StatusMessage statusMessage, String action) { private boolean validateStatusMessage(StatusMessage statusMessage, String action) {
log.info("Received response {} from subprocess while waiting for {}" log.info(
+ " while processing shard {}", statusMessage, action, initializationInput.shardId()); "Received response {} from subprocess while waiting for {}" + " while processing shard {}",
return !(statusMessage == null || statusMessage.getResponseFor() == null || !statusMessage.getResponseFor() statusMessage,
.equals(action)); action,
initializationInput.shardId());
return !(statusMessage == null
|| statusMessage.getResponseFor() == null
|| !statusMessage.getResponseFor().equals(action));
} }
/** /**
@ -274,13 +285,12 @@ class MultiLangProtocol {
} }
return this.messageWriter.writeCheckpointMessageWithError(sequenceNumber, subSequenceNumber, null); return this.messageWriter.writeCheckpointMessageWithError(sequenceNumber, subSequenceNumber, null);
} else { } else {
String message = String message = String.format(
String.format("Was asked to checkpoint at %s but no checkpointer was provided for shard %s", "Was asked to checkpoint at %s but no checkpointer was provided for shard %s",
sequenceNumber, initializationInput.shardId()); sequenceNumber, initializationInput.shardId());
log.error(message); log.error(message);
return this.messageWriter.writeCheckpointMessageWithError(sequenceNumber, subSequenceNumber, return this.messageWriter.writeCheckpointMessageWithError(
new InvalidStateException( sequenceNumber, subSequenceNumber, new InvalidStateException(message));
message));
} }
} catch (Throwable t) { } catch (Throwable t) {
return this.messageWriter.writeCheckpointMessageWithError(sequenceNumber, subSequenceNumber, t); return this.messageWriter.writeCheckpointMessageWithError(sequenceNumber, subSequenceNumber, t);
@ -288,8 +298,8 @@ class MultiLangProtocol {
} }
private String logCheckpointMessage(String sequenceNumber, Long subSequenceNumber) { private String logCheckpointMessage(String sequenceNumber, Long subSequenceNumber) {
return String.format("Attempting to checkpoint shard %s @ sequence number %s, and sub sequence number %s", return String.format(
"Attempting to checkpoint shard %s @ sequence number %s, and sub sequence number %s",
initializationInput.shardId(), sequenceNumber, subSequenceNumber); initializationInput.shardId(), sequenceNumber, subSequenceNumber);
} }
} }

View file

@ -17,11 +17,10 @@ package software.amazon.kinesis.multilang;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration; import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
import software.amazon.kinesis.processor.ShardRecordProcessorFactory;
import software.amazon.kinesis.processor.ShardRecordProcessor; import software.amazon.kinesis.processor.ShardRecordProcessor;
import software.amazon.kinesis.processor.ShardRecordProcessorFactory;
/** /**
* Creates {@link MultiLangShardRecordProcessor}'s. * Creates {@link MultiLangShardRecordProcessor}'s.
@ -43,8 +42,8 @@ public class MultiLangRecordProcessorFactory implements ShardRecordProcessorFact
* @param command The command that will do processing for this factory's record processors. * @param command The command that will do processing for this factory's record processors.
* @param executorService An executor service to use while processing inputs and outputs of the child process. * @param executorService An executor service to use while processing inputs and outputs of the child process.
*/ */
public MultiLangRecordProcessorFactory(String command, ExecutorService executorService, public MultiLangRecordProcessorFactory(
MultiLangDaemonConfiguration configuration) { String command, ExecutorService executorService, MultiLangDaemonConfiguration configuration) {
this(command, executorService, new ObjectMapper(), configuration); this(command, executorService, new ObjectMapper(), configuration);
} }
@ -53,8 +52,11 @@ public class MultiLangRecordProcessorFactory implements ShardRecordProcessorFact
* @param executorService An executor service to use while processing inputs and outputs of the child process. * @param executorService An executor service to use while processing inputs and outputs of the child process.
* @param objectMapper An object mapper used to convert messages to json to be written to the child process * @param objectMapper An object mapper used to convert messages to json to be written to the child process
*/ */
public MultiLangRecordProcessorFactory(String command, ExecutorService executorService, ObjectMapper objectMapper, public MultiLangRecordProcessorFactory(
MultiLangDaemonConfiguration configuration) { String command,
ExecutorService executorService,
ObjectMapper objectMapper,
MultiLangDaemonConfiguration configuration) {
this.command = command; this.command = command;
this.commandArray = command.split(COMMAND_DELIMETER_REGEX); this.commandArray = command.split(COMMAND_DELIMETER_REGEX);
this.executorService = executorService; this.executorService = executorService;
@ -68,8 +70,8 @@ public class MultiLangRecordProcessorFactory implements ShardRecordProcessorFact
/* /*
* Giving ProcessBuilder the command as an array of Strings allows users to specify command line arguments. * Giving ProcessBuilder the command as an array of Strings allows users to specify command line arguments.
*/ */
return new MultiLangShardRecordProcessor(new ProcessBuilder(commandArray), executorService, this.objectMapper, return new MultiLangShardRecordProcessor(
this.configuration); new ProcessBuilder(commandArray), executorService, this.objectMapper, this.configuration);
} }
String[] getCommandArray() { String[] getCommandArray() {

View file

@ -22,7 +22,6 @@ import java.util.concurrent.Future;
import java.util.function.Function; import java.util.function.Function;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import software.amazon.kinesis.lifecycle.events.InitializationInput; import software.amazon.kinesis.lifecycle.events.InitializationInput;
import software.amazon.kinesis.lifecycle.events.LeaseLostInput; import software.amazon.kinesis.lifecycle.events.LeaseLostInput;
@ -156,8 +155,10 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor {
if (ProcessState.ACTIVE.equals(this.state)) { if (ProcessState.ACTIVE.equals(this.state)) {
stopProcessing("Encountered an error while trying to shutdown child process", t); stopProcessing("Encountered an error while trying to shutdown child process", t);
} else { } else {
stopProcessing("Encountered an error during shutdown," stopProcessing(
+ " but it appears the processor has already been shutdown", t); "Encountered an error during shutdown,"
+ " but it appears the processor has already been shutdown",
t);
} }
} }
} }
@ -166,12 +167,13 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor {
* Used to tell whether the processor has been shutdown already. * Used to tell whether the processor has been shutdown already.
*/ */
private enum ProcessState { private enum ProcessState {
ACTIVE, SHUTDOWN ACTIVE,
SHUTDOWN
} }
/** /**
* Constructor. * Constructor.
* *
* @param processBuilder * @param processBuilder
* Provides process builder functionality. * Provides process builder functionality.
* @param executorService * @param executorService
@ -179,15 +181,24 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor {
* @param objectMapper * @param objectMapper
* An obejct mapper. * An obejct mapper.
*/ */
MultiLangShardRecordProcessor(ProcessBuilder processBuilder, ExecutorService executorService, MultiLangShardRecordProcessor(
ObjectMapper objectMapper, MultiLangDaemonConfiguration configuration) { ProcessBuilder processBuilder,
this(processBuilder, executorService, objectMapper, new MessageWriter(), new MessageReader(), ExecutorService executorService,
new DrainChildSTDERRTask(), configuration); ObjectMapper objectMapper,
MultiLangDaemonConfiguration configuration) {
this(
processBuilder,
executorService,
objectMapper,
new MessageWriter(),
new MessageReader(),
new DrainChildSTDERRTask(),
configuration);
} }
/** /**
* Note: This constructor has package level access solely for testing purposes. * Note: This constructor has package level access solely for testing purposes.
* *
* @param processBuilder * @param processBuilder
* Provides the child process for this record processor * Provides the child process for this record processor
* @param executorService * @param executorService
@ -201,9 +212,14 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor {
* @param readSTDERRTask * @param readSTDERRTask
* Error reader to read from child process's stderr * Error reader to read from child process's stderr
*/ */
MultiLangShardRecordProcessor(ProcessBuilder processBuilder, ExecutorService executorService, ObjectMapper objectMapper, MultiLangShardRecordProcessor(
MessageWriter messageWriter, MessageReader messageReader, DrainChildSTDERRTask readSTDERRTask, ProcessBuilder processBuilder,
MultiLangDaemonConfiguration configuration) { ExecutorService executorService,
ObjectMapper objectMapper,
MessageWriter messageWriter,
MessageReader messageReader,
DrainChildSTDERRTask readSTDERRTask,
MultiLangDaemonConfiguration configuration) {
this.executorService = executorService; this.executorService = executorService;
this.processBuilder = processBuilder; this.processBuilder = processBuilder;
this.objectMapper = objectMapper; this.objectMapper = objectMapper;
@ -268,7 +284,7 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor {
/** /**
* Convenience method used by {@link #childProcessShutdownSequence()} to drain the STDIN and STDERR of the child * Convenience method used by {@link #childProcessShutdownSequence()} to drain the STDIN and STDERR of the child
* process. * process.
* *
* @param future A future to wait on. * @param future A future to wait on.
* @param whatThisFutureIsDoing What that future is doing while we wait. * @param whatThisFutureIsDoing What that future is doing while we wait.
*/ */
@ -283,7 +299,7 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor {
/** /**
* Convenience method for logging and safely shutting down so that we don't throw an exception up to the KCL on * Convenience method for logging and safely shutting down so that we don't throw an exception up to the KCL on
* accident. * accident.
* *
* @param message The reason we are stopping processing. * @param message The reason we are stopping processing.
* @param reason An exception that caused us to want to stop processing. * @param reason An exception that caused us to want to stop processing.
*/ */
@ -309,7 +325,7 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor {
/** /**
* The {@link ProcessBuilder} class is final so not easily mocked. We wrap the only interaction we have with it in * The {@link ProcessBuilder} class is final so not easily mocked. We wrap the only interaction we have with it in
* this package level method to permit unit testing. * this package level method to permit unit testing.
* *
* @return The process started by processBuilder * @return The process started by processBuilder
* @throws IOException If the process can't be started. * @throws IOException If the process can't be started.
*/ */

View file

@ -19,7 +19,6 @@ import java.util.Map;
import com.amazonaws.regions.Regions; import com.amazonaws.regions.Regions;
import com.google.common.base.CaseFormat; import com.google.common.base.CaseFormat;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.Getter; import lombok.Getter;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -91,7 +90,6 @@ public enum NestedPropertyKey {
processor.acceptExternalId(externalId); processor.acceptExternalId(externalId);
} }
}, },
; ;
/** /**
@ -141,5 +139,4 @@ public enum NestedPropertyKey {
} }
} }
} }
} }

View file

@ -50,5 +50,4 @@ public interface NestedPropertyProcessor {
* @param externalId external id used in the service call used to retrieve session credentials * @param externalId external id used in the service call used to retrieve session credentials
*/ */
void acceptExternalId(String externalId); void acceptExternalId(String externalId);
} }

View file

@ -24,7 +24,6 @@ import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration;
import com.amazonaws.regions.Regions; import com.amazonaws.regions.Regions;
import com.amazonaws.services.securitytoken.AWSSecurityTokenService; import com.amazonaws.services.securitytoken.AWSSecurityTokenService;
import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient; import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient;
import software.amazon.kinesis.multilang.NestedPropertyKey; import software.amazon.kinesis.multilang.NestedPropertyKey;
import software.amazon.kinesis.multilang.NestedPropertyProcessor; import software.amazon.kinesis.multilang.NestedPropertyProcessor;
@ -47,8 +46,8 @@ public class KclSTSAssumeRoleSessionCredentialsProvider
this(params[0], params[1], Arrays.copyOfRange(params, 2, params.length)); this(params[0], params[1], Arrays.copyOfRange(params, 2, params.length));
} }
public KclSTSAssumeRoleSessionCredentialsProvider(final String roleArn, final String roleSessionName, public KclSTSAssumeRoleSessionCredentialsProvider(
final String... params) { final String roleArn, final String roleSessionName, final String... params) {
builder = new Builder(roleArn, roleSessionName); builder = new Builder(roleArn, roleSessionName);
NestedPropertyKey.parse(this, params); NestedPropertyKey.parse(this, params);
provider = builder.build(); provider = builder.build();
@ -75,9 +74,8 @@ public class KclSTSAssumeRoleSessionCredentialsProvider
@Override @Override
public void acceptEndpointRegion(final Regions region) { public void acceptEndpointRegion(final Regions region) {
final AWSSecurityTokenService stsClient = AWSSecurityTokenServiceClient.builder() final AWSSecurityTokenService stsClient =
.withRegion(region) AWSSecurityTokenServiceClient.builder().withRegion(region).build();
.build();
builder.withStsClient(stsClient); builder.withStsClient(stsClient);
} }
@ -85,5 +83,4 @@ public class KclSTSAssumeRoleSessionCredentialsProvider
public void acceptExternalId(final String externalId) { public void acceptExternalId(final String externalId) {
builder.withExternalId(externalId); builder.withExternalId(externalId);
} }
}
}

View file

@ -37,8 +37,7 @@ class AWSCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode
/** /**
* Constructor. * Constructor.
*/ */
AWSCredentialsProviderPropertyValueDecoder() { AWSCredentialsProviderPropertyValueDecoder() {}
}
/** /**
* Get AWSCredentialsProvider property. * Get AWSCredentialsProvider property.
@ -104,9 +103,8 @@ class AWSCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode
if (provider == null) { if (provider == null) {
// attempt to invoke a public varargs/array constructor of FooClass(String[]) // attempt to invoke a public varargs/array constructor of FooClass(String[])
provider = constructProvider(providerName, () -> provider = constructProvider(providerName, () -> clazz.getConstructor(String[].class)
clazz.getConstructor(String[].class).newInstance((Object) varargs) .newInstance((Object) varargs));
);
} }
} }
@ -138,24 +136,26 @@ class AWSCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode
private static List<String> getPossibleFullClassNames(final String provider) { private static List<String> getPossibleFullClassNames(final String provider) {
return Stream.of( return Stream.of(
// Customer provides a short name of common providers in com.amazonaws.auth package // Customer provides a short name of common providers in com.amazonaws.auth package
// (e.g., any classes implementing the AWSCredentialsProvider interface) // (e.g., any classes implementing the AWSCredentialsProvider interface)
// @see http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/AWSCredentialsProvider.html // @see
"com.amazonaws.auth.", // http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/AWSCredentialsProvider.html
"com.amazonaws.auth.",
// Customer provides a short name of a provider offered by this multi-lang package // Customer provides a short name of a provider offered by this multi-lang package
"software.amazon.kinesis.multilang.auth.", "software.amazon.kinesis.multilang.auth.",
// Customer provides a fully-qualified provider name, or a custom credentials provider // Customer provides a fully-qualified provider name, or a custom credentials provider
// (e.g., com.amazonaws.auth.ClasspathFileCredentialsProvider, org.mycompany.FooProvider) // (e.g., com.amazonaws.auth.ClasspathFileCredentialsProvider, org.mycompany.FooProvider)
"" "")
).map(prefix -> prefix + provider).collect(Collectors.toList()); .map(prefix -> prefix + provider)
.collect(Collectors.toList());
} }
@FunctionalInterface @FunctionalInterface
private interface CredentialsProviderConstructor<T extends AWSCredentialsProvider> { private interface CredentialsProviderConstructor<T extends AWSCredentialsProvider> {
T construct() throws IllegalAccessException, InstantiationException, T construct()
InvocationTargetException, NoSuchMethodException; throws IllegalAccessException, InstantiationException, InvocationTargetException, NoSuchMethodException;
} }
/** /**
@ -179,5 +179,4 @@ class AWSCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode
} }
return null; return null;
} }
} }

View file

@ -31,7 +31,7 @@ import org.apache.commons.lang3.StringUtils;
public class BuilderDynaBean implements DynaBean { public class BuilderDynaBean implements DynaBean {
private static final String[] CLASS_NAME_JOINERS = { ClassUtils.PACKAGE_SEPARATOR, ClassUtils.INNER_CLASS_SEPARATOR }; private static final String[] CLASS_NAME_JOINERS = {ClassUtils.PACKAGE_SEPARATOR, ClassUtils.INNER_CLASS_SEPARATOR};
static final String NO_MAP_ACCESS_SUPPORT = "Map access isn't supported"; static final String NO_MAP_ACCESS_SUPPORT = "Map access isn't supported";
private Class<?> destinedClass; private Class<?> destinedClass;
@ -51,13 +51,19 @@ public class BuilderDynaBean implements DynaBean {
this(destinedClass, convertUtilsBean, null, Arrays.asList(classPrefixSearchList)); this(destinedClass, convertUtilsBean, null, Arrays.asList(classPrefixSearchList));
} }
public BuilderDynaBean(Class<?> destinedClass, ConvertUtilsBean convertUtilsBean, public BuilderDynaBean(
Function<String, ?> emptyPropertyHandler, String... classPrefixSearchList) { Class<?> destinedClass,
ConvertUtilsBean convertUtilsBean,
Function<String, ?> emptyPropertyHandler,
String... classPrefixSearchList) {
this(destinedClass, convertUtilsBean, emptyPropertyHandler, Arrays.asList(classPrefixSearchList)); this(destinedClass, convertUtilsBean, emptyPropertyHandler, Arrays.asList(classPrefixSearchList));
} }
public BuilderDynaBean(Class<?> destinedClass, ConvertUtilsBean convertUtilsBean, public BuilderDynaBean(
Function<String, ?> emptyPropertyHandler, List<String> classPrefixSearchList) { Class<?> destinedClass,
ConvertUtilsBean convertUtilsBean,
Function<String, ?> emptyPropertyHandler,
List<String> classPrefixSearchList) {
this.convertUtilsBean = convertUtilsBean; this.convertUtilsBean = convertUtilsBean;
this.classPrefixSearchList = classPrefixSearchList; this.classPrefixSearchList = classPrefixSearchList;
this.emptyPropertyHandler = emptyPropertyHandler; this.emptyPropertyHandler = emptyPropertyHandler;
@ -102,7 +108,6 @@ public class BuilderDynaBean implements DynaBean {
// Ignored // Ignored
// //
} }
} }
} }
} }
@ -214,8 +219,10 @@ public class BuilderDynaBean implements DynaBean {
validateCanBuildOrCreate(); validateCanBuildOrCreate();
List<TypeTag> types = dynaBeanBuilderSupport.getProperty(name); List<TypeTag> types = dynaBeanBuilderSupport.getProperty(name);
if (types.size() > 1) { if (types.size() > 1) {
Optional<TypeTag> arrayType = types.stream().filter(t -> t.type.isArray()).findFirst(); Optional<TypeTag> arrayType =
return arrayType.map(t -> new DynaProperty(name, t.type, t.type.getComponentType())) types.stream().filter(t -> t.type.isArray()).findFirst();
return arrayType
.map(t -> new DynaProperty(name, t.type, t.type.getComponentType()))
.orElseGet(() -> new DynaProperty(name)); .orElseGet(() -> new DynaProperty(name));
} else { } else {
TypeTag type = types.get(0); TypeTag type = types.get(0);
@ -232,7 +239,8 @@ public class BuilderDynaBean implements DynaBean {
@Override @Override
public DynaProperty[] getDynaProperties() { public DynaProperty[] getDynaProperties() {
validateCanBuildOrCreate(); validateCanBuildOrCreate();
return dynaBeanBuilderSupport.getPropertyNames().stream().map(this::getDynaProperty) return dynaBeanBuilderSupport.getPropertyNames().stream()
.map(this::getDynaProperty)
.toArray(DynaProperty[]::new); .toArray(DynaProperty[]::new);
} }

View file

@ -28,21 +28,21 @@ public @interface ConfigurationSettable {
/** /**
* Which builder this option applies to * Which builder this option applies to
* *
* @return the class of the builder to use * @return the class of the builder to use
*/ */
Class<?> configurationClass(); Class<?> configurationClass();
/** /**
* The method name on the builder, defaults to the fieldName * The method name on the builder, defaults to the fieldName
* *
* @return the name of the method or null to use the default * @return the name of the method or null to use the default
*/ */
String methodName() default ""; String methodName() default "";
/** /**
* If the type is actually an optional value this will enable conversions * If the type is actually an optional value this will enable conversions
* *
* @return true if the value should be wrapped by an optional * @return true if the value should be wrapped by an optional
*/ */
boolean convertToOptional() default false; boolean convertToOptional() default false;

View file

@ -23,13 +23,11 @@ import java.util.Map;
import java.util.Optional; import java.util.Optional;
import java.util.Set; import java.util.Set;
import com.google.common.base.Defaults;
import lombok.NonNull;
import org.apache.commons.lang3.ClassUtils; import org.apache.commons.lang3.ClassUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import com.google.common.base.Defaults;
import lombok.NonNull;
public class ConfigurationSettableUtils { public class ConfigurationSettableUtils {
public static <T> T resolveFields(@NonNull Object source, @NonNull T configObject) { public static <T> T resolveFields(@NonNull Object source, @NonNull T configObject) {
@ -40,8 +38,8 @@ public class ConfigurationSettableUtils {
return configObject; return configObject;
} }
public static void resolveFields(Object source, Map<Class<?>, Object> configObjects, Set<Class<?>> restrictTo, public static void resolveFields(
Set<Class<?>> skipIf) { Object source, Map<Class<?>, Object> configObjects, Set<Class<?>> restrictTo, Set<Class<?>> skipIf) {
for (Field field : source.getClass().getDeclaredFields()) { for (Field field : source.getClass().getDeclaredFields()) {
for (ConfigurationSettable b : field.getAnnotationsByType(ConfigurationSettable.class)) { for (ConfigurationSettable b : field.getAnnotationsByType(ConfigurationSettable.class)) {
if (restrictTo != null && !restrictTo.contains(b.configurationClass())) { if (restrictTo != null && !restrictTo.contains(b.configurationClass())) {
@ -70,9 +68,11 @@ public class ConfigurationSettableUtils {
value = Optional.of(value); value = Optional.of(value);
} }
if (ClassUtils.isPrimitiveOrWrapper(value.getClass())) { if (ClassUtils.isPrimitiveOrWrapper(value.getClass())) {
Class<?> primitiveType = field.getType().isPrimitive() ? field.getType() Class<?> primitiveType = field.getType().isPrimitive()
? field.getType()
: ClassUtils.wrapperToPrimitive(field.getType()); : ClassUtils.wrapperToPrimitive(field.getType());
Class<?> wrapperType = !field.getType().isPrimitive() ? field.getType() Class<?> wrapperType = !field.getType().isPrimitive()
? field.getType()
: ClassUtils.primitiveToWrapper(field.getType()); : ClassUtils.primitiveToWrapper(field.getType());
try { try {

View file

@ -26,8 +26,7 @@ public class DatePropertyValueDecoder implements IPropertyValueDecoder<Date> {
/** /**
* Constructor. * Constructor.
*/ */
DatePropertyValueDecoder() { DatePropertyValueDecoder() {}
}
/** /**
* @param value property value as String * @param value property value as String
@ -49,5 +48,4 @@ public class DatePropertyValueDecoder implements IPropertyValueDecoder<Date> {
public List<Class<Date>> getSupportedTypes() { public List<Class<Date>> getSupportedTypes() {
return Arrays.asList(Date.class); return Arrays.asList(Date.class);
} }
} }

View file

@ -29,11 +29,10 @@ import java.util.function.Consumer;
import java.util.function.Function; import java.util.function.Function;
import java.util.function.Supplier; import java.util.function.Supplier;
import org.apache.commons.beanutils.ConvertUtilsBean;
import org.apache.commons.lang3.ClassUtils;
import com.google.common.collect.HashMultimap; import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap; import com.google.common.collect.Multimap;
import org.apache.commons.beanutils.ConvertUtilsBean;
import org.apache.commons.lang3.ClassUtils;
class DynaBeanBuilderSupport { class DynaBeanBuilderSupport {
@ -48,8 +47,8 @@ class DynaBeanBuilderSupport {
private final Multimap<String, TypeTag> properties = HashMultimap.create(); private final Multimap<String, TypeTag> properties = HashMultimap.create();
private final Map<String, Object> values = new HashMap<>(); private final Map<String, Object> values = new HashMap<>();
DynaBeanBuilderSupport(Class<?> destinedClass, ConvertUtilsBean convertUtilsBean, DynaBeanBuilderSupport(
List<String> classPrefixSearchList) { Class<?> destinedClass, ConvertUtilsBean convertUtilsBean, List<String> classPrefixSearchList) {
this.destinedClass = destinedClass; this.destinedClass = destinedClass;
this.convertUtilsBean = convertUtilsBean; this.convertUtilsBean = convertUtilsBean;
this.classPrefixSearchList = classPrefixSearchList; this.classPrefixSearchList = classPrefixSearchList;
@ -103,11 +102,12 @@ class DynaBeanBuilderSupport {
private Object createForProperty(String name) { private Object createForProperty(String name) {
Optional<TypeTag> type = properties.get(name).stream().findFirst(); Optional<TypeTag> type = properties.get(name).stream().findFirst();
return type.map(t -> { return type.map(t -> {
if (DynaBeanBuilderUtils.isBuilderOrCreate(t.type) || !t.hasConverter) { if (DynaBeanBuilderUtils.isBuilderOrCreate(t.type) || !t.hasConverter) {
return new BuilderDynaBean(t.type, convertUtilsBean, null, classPrefixSearchList); return new BuilderDynaBean(t.type, convertUtilsBean, null, classPrefixSearchList);
} }
return null; return null;
}).orElse(null); })
.orElse(null);
} }
boolean hasValue(String name) { boolean hasValue(String name) {
@ -157,8 +157,11 @@ class DynaBeanBuilderSupport {
void set(String name, Object value) { void set(String name, Object value) {
if (value instanceof String && properties.get(name).stream().anyMatch(t -> t.type.isEnum())) { if (value instanceof String && properties.get(name).stream().anyMatch(t -> t.type.isEnum())) {
TypeTag typeTag = properties.get(name).stream().filter(t -> t.type.isEnum()).findFirst().orElseThrow( TypeTag typeTag = properties.get(name).stream()
() -> new IllegalStateException("Expected enum type for " + name + ", but couldn't find it.")); .filter(t -> t.type.isEnum())
.findFirst()
.orElseThrow(() ->
new IllegalStateException("Expected enum type for " + name + ", but couldn't find it."));
Class<? extends Enum> enumClass = (Class<? extends Enum>) typeTag.type; Class<? extends Enum> enumClass = (Class<? extends Enum>) typeTag.type;
values.put(name, Enum.valueOf(enumClass, value.toString())); values.put(name, Enum.valueOf(enumClass, value.toString()));
} else { } else {
@ -174,9 +177,11 @@ class DynaBeanBuilderSupport {
private Object getArgument(Map.Entry<String, Object> setValue) { private Object getArgument(Map.Entry<String, Object> setValue) {
Object argument = setValue.getValue(); Object argument = setValue.getValue();
if (argument instanceof Object[]) { if (argument instanceof Object[]) {
TypeTag arrayType = properties.get(setValue.getKey()).stream().filter(t -> t.type.isArray()).findFirst() TypeTag arrayType = properties.get(setValue.getKey()).stream()
.orElseThrow(() -> new IllegalStateException(String .filter(t -> t.type.isArray())
.format("Received Object[] for %s but can't find corresponding type", setValue.getKey()))); .findFirst()
.orElseThrow(() -> new IllegalStateException(String.format(
"Received Object[] for %s but can't find corresponding type", setValue.getKey())));
Object[] arrayValues = (Object[]) argument; Object[] arrayValues = (Object[]) argument;
Object[] destination = (Object[]) Array.newInstance(arrayType.type.getComponentType(), arrayValues.length); Object[] destination = (Object[]) Array.newInstance(arrayType.type.getComponentType(), arrayValues.length);
@ -212,10 +217,12 @@ class DynaBeanBuilderSupport {
for (Map.Entry<String, Object> setValue : values.entrySet()) { for (Map.Entry<String, Object> setValue : values.entrySet()) {
Object argument = getArgument(setValue); Object argument = getArgument(setValue);
Method mutator = properties.get(setValue.getKey()).stream() Method mutator = properties.get(setValue.getKey()).stream()
.filter(t -> ClassUtils.isAssignable(argument.getClass(), t.type)).findFirst() .filter(t -> ClassUtils.isAssignable(argument.getClass(), t.type))
.map(a -> a.builderMethod).orElseThrow( .findFirst()
() -> new IllegalStateException(String.format("Unable to find mutator for %s of type %s", .map(a -> a.builderMethod)
setValue.getKey(), argument.getClass().getName()))); .orElseThrow(() -> new IllegalStateException(String.format(
"Unable to find mutator for %s of type %s",
setValue.getKey(), argument.getClass().getName())));
try { try {
source = mutator.invoke(source, argument); source = mutator.invoke(source, argument);
} catch (IllegalAccessException | InvocationTargetException e) { } catch (IllegalAccessException | InvocationTargetException e) {
@ -236,7 +243,6 @@ class DynaBeanBuilderSupport {
} catch (IllegalAccessException | NoSuchMethodException | InvocationTargetException e) { } catch (IllegalAccessException | NoSuchMethodException | InvocationTargetException e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
} }
Collection<String> getPropertyNames() { Collection<String> getPropertyNames() {
@ -249,5 +255,4 @@ class DynaBeanBuilderSupport {
} }
return new ArrayList<>(properties.get(name)); return new ArrayList<>(properties.get(name));
} }
} }

View file

@ -30,8 +30,8 @@ class DynaBeanCreateSupport {
private final List<TypeTag> createTypes = new ArrayList<>(); private final List<TypeTag> createTypes = new ArrayList<>();
private Object[] createValues = null; private Object[] createValues = null;
DynaBeanCreateSupport(Class<?> destinedClass, ConvertUtilsBean convertUtilsBean, DynaBeanCreateSupport(
List<String> classPrefixSearchList) { Class<?> destinedClass, ConvertUtilsBean convertUtilsBean, List<String> classPrefixSearchList) {
this.destinedClass = destinedClass; this.destinedClass = destinedClass;
this.convertUtilsBean = convertUtilsBean; this.convertUtilsBean = convertUtilsBean;
this.classPrefixSearchList = classPrefixSearchList; this.classPrefixSearchList = classPrefixSearchList;
@ -58,8 +58,8 @@ class DynaBeanCreateSupport {
Object build() { Object build() {
Method createMethod = DynaBeanBuilderUtils.getMethod(destinedClass, "create", Method createMethod = DynaBeanBuilderUtils.getMethod(
createTypes.stream().map(t -> t.type).toArray(i -> new Class<?>[i])); destinedClass, "create", createTypes.stream().map(t -> t.type).toArray(i -> new Class<?>[i]));
Object arguments[] = new Object[createValues.length]; Object arguments[] = new Object[createValues.length];
for (int i = 0; i < createValues.length; ++i) { for (int i = 0; i < createValues.length; ++i) {
if (createValues[i] instanceof BuilderDynaBean) { if (createValues[i] instanceof BuilderDynaBean) {
@ -77,8 +77,8 @@ class DynaBeanCreateSupport {
return createValues[index]; return createValues[index];
} else { } else {
if (createValues[index] == null) { if (createValues[index] == null) {
createValues[index] = new BuilderDynaBean(createTypes.get(index).type, convertUtilsBean, null, createValues[index] = new BuilderDynaBean(
classPrefixSearchList); createTypes.get(index).type, convertUtilsBean, null, classPrefixSearchList);
} }
return createValues[index]; return createValues[index];
} }
@ -89,13 +89,11 @@ class DynaBeanCreateSupport {
public void set(String name, int index, Object value) { public void set(String name, int index, Object value) {
if (StringUtils.isEmpty(name)) { if (StringUtils.isEmpty(name)) {
if (index >= createValues.length) { if (index >= createValues.length) {
throw new IllegalArgumentException( throw new IllegalArgumentException(String.format(
String.format("%d exceeds the maximum number of arguments (%d) for %s", index, "%d exceeds the maximum number of arguments (%d) for %s",
createValues.length, destinedClass.getName())); index, createValues.length, destinedClass.getName()));
} }
createValues[index] = value; createValues[index] = value;
} }
} }
} }

View file

@ -26,21 +26,28 @@ public class FanoutConfigBean implements RetrievalConfigBuilder {
@ConfigurationSettable(configurationClass = FanOutConfig.class) @ConfigurationSettable(configurationClass = FanOutConfig.class)
private int maxDescribeStreamSummaryRetries; private int maxDescribeStreamSummaryRetries;
@ConfigurationSettable(configurationClass = FanOutConfig.class) @ConfigurationSettable(configurationClass = FanOutConfig.class)
private String consumerArn; private String consumerArn;
@ConfigurationSettable(configurationClass = FanOutConfig.class) @ConfigurationSettable(configurationClass = FanOutConfig.class)
private String consumerName; private String consumerName;
@ConfigurationSettable(configurationClass = FanOutConfig.class) @ConfigurationSettable(configurationClass = FanOutConfig.class)
private int maxDescribeStreamConsumerRetries; private int maxDescribeStreamConsumerRetries;
@ConfigurationSettable(configurationClass = FanOutConfig.class) @ConfigurationSettable(configurationClass = FanOutConfig.class)
private int registerStreamConsumerRetries; private int registerStreamConsumerRetries;
@ConfigurationSettable(configurationClass = FanOutConfig.class) @ConfigurationSettable(configurationClass = FanOutConfig.class)
private long retryBackoffMillis; private long retryBackoffMillis;
@Override @Override
public FanOutConfig build(KinesisAsyncClient kinesisAsyncClient, MultiLangDaemonConfiguration parent) { public FanOutConfig build(KinesisAsyncClient kinesisAsyncClient, MultiLangDaemonConfiguration parent) {
return ConfigurationSettableUtils.resolveFields(this, new FanOutConfig(kinesisAsyncClient).applicationName(parent.getApplicationName()) return ConfigurationSettableUtils.resolveFields(
.streamName(parent.getStreamName())); this,
new FanOutConfig(kinesisAsyncClient)
.applicationName(parent.getApplicationName())
.streamName(parent.getStreamName()));
} }
} }

View file

@ -18,13 +18,13 @@ import java.util.List;
/** /**
* This class captures the concept of decoding a property value to a particular Java type. * This class captures the concept of decoding a property value to a particular Java type.
* *
* @param <T> * @param <T>
*/ */
interface IPropertyValueDecoder<T> { interface IPropertyValueDecoder<T> {
/** /**
* Get the value that was read from a configuration file and convert it to some type. * Get the value that was read from a configuration file and convert it to some type.
* *
* @param propertyValue property string value that needs to be decoded. * @param propertyValue property string value that needs to be decoded.
* @return property value in type T * @return property value in type T
*/ */
@ -32,7 +32,7 @@ interface IPropertyValueDecoder<T> {
/** /**
* Get a list of supported types for this class. * Get a list of supported types for this class.
* *
* @return list of supported classes. * @return list of supported classes.
*/ */
List<Class<T>> getSupportedTypes(); List<Class<T>> getSupportedTypes();

View file

@ -25,8 +25,7 @@ class IntegerPropertyValueDecoder implements IPropertyValueDecoder<Integer> {
/** /**
* Constructor. * Constructor.
*/ */
IntegerPropertyValueDecoder() { IntegerPropertyValueDecoder() {}
}
/** /**
* @param value property value as String * @param value property value as String

View file

@ -19,10 +19,9 @@ import java.io.InputStream;
import java.lang.reflect.InvocationTargetException; import java.lang.reflect.InvocationTargetException;
import java.util.Properties; import java.util.Properties;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.beanutils.BeanUtilsBean; import org.apache.commons.beanutils.BeanUtilsBean;
import org.apache.commons.beanutils.ConvertUtilsBean; import org.apache.commons.beanutils.ConvertUtilsBean;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.Validate; import org.apache.commons.lang3.Validate;
import software.amazon.awssdk.arns.Arn; import software.amazon.awssdk.arns.Arn;
import software.amazon.kinesis.common.StreamIdentifier; import software.amazon.kinesis.common.StreamIdentifier;
@ -71,16 +70,21 @@ public class KinesisClientLibConfigurator {
Validate.notBlank(configuration.getApplicationName(), "Application name is required"); Validate.notBlank(configuration.getApplicationName(), "Application name is required");
if (configuration.getStreamArn() != null && !configuration.getStreamArn().trim().isEmpty()) { if (configuration.getStreamArn() != null
&& !configuration.getStreamArn().trim().isEmpty()) {
final Arn streamArnObj = Arn.fromString(configuration.getStreamArn()); final Arn streamArnObj = Arn.fromString(configuration.getStreamArn());
StreamIdentifier.validateArn(streamArnObj); StreamIdentifier.validateArn(streamArnObj);
//Parse out the stream Name from the Arn (and/or override existing value for Stream Name) // Parse out the stream Name from the Arn (and/or override existing value for Stream Name)
final String streamNameFromArn = streamArnObj.resource().resource(); final String streamNameFromArn = streamArnObj.resource().resource();
configuration.setStreamName(streamNameFromArn); configuration.setStreamName(streamNameFromArn);
} }
Validate.notBlank(configuration.getStreamName(), "Stream name or Stream Arn is required. Stream Arn takes precedence if both are passed in."); Validate.notBlank(
Validate.isTrue(configuration.getKinesisCredentialsProvider().isDirty(), "A basic set of AWS credentials must be provided"); configuration.getStreamName(),
"Stream name or Stream Arn is required. Stream Arn takes precedence if both are passed in.");
Validate.isTrue(
configuration.getKinesisCredentialsProvider().isDirty(),
"A basic set of AWS credentials must be provided");
return configuration; return configuration;
} }
@ -106,5 +110,4 @@ public class KinesisClientLibConfigurator {
} }
return getConfiguration(properties); return getConfiguration(properties);
} }
} }

View file

@ -27,17 +27,16 @@ import java.util.Set;
import java.util.UUID; import java.util.UUID;
import java.util.function.Function; import java.util.function.Function;
import org.apache.commons.beanutils.BeanUtilsBean;
import org.apache.commons.beanutils.ConvertUtilsBean;
import org.apache.commons.beanutils.Converter;
import org.apache.commons.beanutils.converters.ArrayConverter;
import org.apache.commons.beanutils.converters.StringConverter;
import lombok.Data; import lombok.Data;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
import lombok.experimental.Delegate; import lombok.experimental.Delegate;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.beanutils.BeanUtilsBean;
import org.apache.commons.beanutils.ConvertUtilsBean;
import org.apache.commons.beanutils.Converter;
import org.apache.commons.beanutils.converters.ArrayConverter;
import org.apache.commons.beanutils.converters.StringConverter;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient; import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient;
@ -74,7 +73,6 @@ public class MultiLangDaemonConfiguration {
private String streamName; private String streamName;
private String streamArn; private String streamArn;
@ConfigurationSettable(configurationClass = ConfigsBuilder.class) @ConfigurationSettable(configurationClass = ConfigsBuilder.class)
private String tableName; private String tableName;
@ -86,22 +84,31 @@ public class MultiLangDaemonConfiguration {
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class) @ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private long failoverTimeMillis; private long failoverTimeMillis;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class) @ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private Boolean enablePriorityLeaseAssignment; private Boolean enablePriorityLeaseAssignment;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class) @ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private long shardSyncIntervalMillis; private long shardSyncIntervalMillis;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class) @ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private boolean cleanupLeasesUponShardCompletion; private boolean cleanupLeasesUponShardCompletion;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class) @ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private boolean ignoreUnexpectedChildShards; private boolean ignoreUnexpectedChildShards;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class) @ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private int maxLeasesForWorker; private int maxLeasesForWorker;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class) @ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private int maxLeasesToStealAtOneTime; private int maxLeasesToStealAtOneTime;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class) @ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private int initialLeaseTableReadCapacity; private int initialLeaseTableReadCapacity;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class) @ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private int initialLeaseTableWriteCapacity; private int initialLeaseTableWriteCapacity;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class, methodName = "initialPositionInStream") @ConfigurationSettable(configurationClass = LeaseManagementConfig.class, methodName = "initialPositionInStream")
@ConfigurationSettable(configurationClass = RetrievalConfig.class) @ConfigurationSettable(configurationClass = RetrievalConfig.class)
private InitialPositionInStreamExtended initialPositionInStreamExtended; private InitialPositionInStreamExtended initialPositionInStreamExtended;
@ -114,14 +121,16 @@ public class MultiLangDaemonConfiguration {
} }
public void setInitialPositionInStream(InitialPositionInStream initialPositionInStream) { public void setInitialPositionInStream(InitialPositionInStream initialPositionInStream) {
this.initialPositionInStreamExtended = InitialPositionInStreamExtended this.initialPositionInStreamExtended =
.newInitialPosition(initialPositionInStream); InitialPositionInStreamExtended.newInitialPosition(initialPositionInStream);
} }
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class) @ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private int maxLeaseRenewalThreads; private int maxLeaseRenewalThreads;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class) @ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private long listShardsBackoffTimeInMillis; private long listShardsBackoffTimeInMillis;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class) @ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private int maxListShardsRetryAttempts; private int maxListShardsRetryAttempts;
@ -131,10 +140,13 @@ public class MultiLangDaemonConfiguration {
@ConfigurationSettable(configurationClass = CoordinatorConfig.class) @ConfigurationSettable(configurationClass = CoordinatorConfig.class)
private long parentShardPollIntervalMillis; private long parentShardPollIntervalMillis;
@ConfigurationSettable(configurationClass = CoordinatorConfig.class) @ConfigurationSettable(configurationClass = CoordinatorConfig.class)
private ShardPrioritization shardPrioritization; private ShardPrioritization shardPrioritization;
@ConfigurationSettable(configurationClass = CoordinatorConfig.class) @ConfigurationSettable(configurationClass = CoordinatorConfig.class)
private boolean skipShardSyncAtWorkerInitializationIfLeasesExist; private boolean skipShardSyncAtWorkerInitializationIfLeasesExist;
@ConfigurationSettable(configurationClass = CoordinatorConfig.class) @ConfigurationSettable(configurationClass = CoordinatorConfig.class)
private long schedulerInitializationBackoffTimeMillis; private long schedulerInitializationBackoffTimeMillis;
@ -143,12 +155,16 @@ public class MultiLangDaemonConfiguration {
@ConfigurationSettable(configurationClass = MetricsConfig.class) @ConfigurationSettable(configurationClass = MetricsConfig.class)
private long metricsBufferTimeMillis; private long metricsBufferTimeMillis;
@ConfigurationSettable(configurationClass = MetricsConfig.class) @ConfigurationSettable(configurationClass = MetricsConfig.class)
private int metricsMaxQueueSize; private int metricsMaxQueueSize;
@ConfigurationSettable(configurationClass = MetricsConfig.class) @ConfigurationSettable(configurationClass = MetricsConfig.class)
private MetricsLevel metricsLevel; private MetricsLevel metricsLevel;
@ConfigurationSettable(configurationClass = LifecycleConfig.class, convertToOptional = true) @ConfigurationSettable(configurationClass = LifecycleConfig.class, convertToOptional = true)
private Long logWarningForTaskAfterMillis; private Long logWarningForTaskAfterMillis;
@ConfigurationSettable(configurationClass = MetricsConfig.class) @ConfigurationSettable(configurationClass = MetricsConfig.class)
private Set<String> metricsEnabledDimensions; private Set<String> metricsEnabledDimensions;
@ -163,6 +179,7 @@ public class MultiLangDaemonConfiguration {
private RetrievalMode retrievalMode = RetrievalMode.DEFAULT; private RetrievalMode retrievalMode = RetrievalMode.DEFAULT;
private final FanoutConfigBean fanoutConfig = new FanoutConfigBean(); private final FanoutConfigBean fanoutConfig = new FanoutConfigBean();
@Delegate(types = PollingConfigBean.PollingConfigBeanDelegate.class) @Delegate(types = PollingConfigBean.PollingConfigBeanDelegate.class)
private final PollingConfigBean pollingConfig = new PollingConfigBean(); private final PollingConfigBean pollingConfig = new PollingConfigBean();
@ -200,61 +217,75 @@ public class MultiLangDaemonConfiguration {
this.utilsBean = utilsBean; this.utilsBean = utilsBean;
this.convertUtilsBean = convertUtilsBean; this.convertUtilsBean = convertUtilsBean;
convertUtilsBean.register(new Converter() { convertUtilsBean.register(
@Override new Converter() {
public <T> T convert(Class<T> type, Object value) { @Override
Date date = new Date(Long.parseLong(value.toString()) * 1000L); public <T> T convert(Class<T> type, Object value) {
return type.cast(InitialPositionInStreamExtended.newInitialPositionAtTimestamp(date)); Date date = new Date(Long.parseLong(value.toString()) * 1000L);
} return type.cast(InitialPositionInStreamExtended.newInitialPositionAtTimestamp(date));
}, InitialPositionInStreamExtended.class); }
},
InitialPositionInStreamExtended.class);
convertUtilsBean.register(new Converter() { convertUtilsBean.register(
@Override new Converter() {
public <T> T convert(Class<T> type, Object value) { @Override
return type.cast(MetricsLevel.valueOf(value.toString().toUpperCase())); public <T> T convert(Class<T> type, Object value) {
} return type.cast(MetricsLevel.valueOf(value.toString().toUpperCase()));
}, MetricsLevel.class); }
},
MetricsLevel.class);
convertUtilsBean.register(new Converter() { convertUtilsBean.register(
@Override new Converter() {
public <T> T convert(Class<T> type, Object value) { @Override
return type.cast(InitialPositionInStream.valueOf(value.toString().toUpperCase())); public <T> T convert(Class<T> type, Object value) {
} return type.cast(
}, InitialPositionInStream.class); InitialPositionInStream.valueOf(value.toString().toUpperCase()));
}
},
InitialPositionInStream.class);
convertUtilsBean.register(new Converter() { convertUtilsBean.register(
@Override new Converter() {
public <T> T convert(Class<T> type, Object value) { @Override
return type.cast(URI.create(value.toString())); public <T> T convert(Class<T> type, Object value) {
} return type.cast(URI.create(value.toString()));
}, URI.class); }
},
URI.class);
convertUtilsBean.register(new Converter() { convertUtilsBean.register(
@Override new Converter() {
public <T> T convert(Class<T> type, Object value) { @Override
return type.cast(RetrievalMode.from(value.toString())); public <T> T convert(Class<T> type, Object value) {
} return type.cast(RetrievalMode.from(value.toString()));
}, RetrievalMode.class); }
},
RetrievalMode.class);
convertUtilsBean.register(new Converter() { convertUtilsBean.register(
@Override new Converter() {
public <T> T convert(final Class<T> type, final Object value) { @Override
return type.cast(Region.of(value.toString())); public <T> T convert(final Class<T> type, final Object value) {
} return type.cast(Region.of(value.toString()));
}, Region.class); }
},
Region.class);
ArrayConverter arrayConverter = new ArrayConverter(String[].class, new StringConverter()); ArrayConverter arrayConverter = new ArrayConverter(String[].class, new StringConverter());
arrayConverter.setDelimiter(','); arrayConverter.setDelimiter(',');
convertUtilsBean.register(arrayConverter, String[].class); convertUtilsBean.register(arrayConverter, String[].class);
AWSCredentialsProviderPropertyValueDecoder oldCredentialsDecoder = new AWSCredentialsProviderPropertyValueDecoder(); AWSCredentialsProviderPropertyValueDecoder oldCredentialsDecoder =
new AWSCredentialsProviderPropertyValueDecoder();
Function<String, ?> converter = s -> new V2CredentialWrapper(oldCredentialsDecoder.decodeValue(s)); Function<String, ?> converter = s -> new V2CredentialWrapper(oldCredentialsDecoder.decodeValue(s));
this.kinesisCredentialsProvider = new BuilderDynaBean(AwsCredentialsProvider.class, convertUtilsBean, this.kinesisCredentialsProvider = new BuilderDynaBean(
converter, CREDENTIALS_DEFAULT_SEARCH_PATH); AwsCredentialsProvider.class, convertUtilsBean, converter, CREDENTIALS_DEFAULT_SEARCH_PATH);
this.dynamoDBCredentialsProvider = new BuilderDynaBean(AwsCredentialsProvider.class, convertUtilsBean, this.dynamoDBCredentialsProvider = new BuilderDynaBean(
converter, CREDENTIALS_DEFAULT_SEARCH_PATH); AwsCredentialsProvider.class, convertUtilsBean, converter, CREDENTIALS_DEFAULT_SEARCH_PATH);
this.cloudWatchCredentialsProvider = new BuilderDynaBean(AwsCredentialsProvider.class, convertUtilsBean, this.cloudWatchCredentialsProvider = new BuilderDynaBean(
converter, CREDENTIALS_DEFAULT_SEARCH_PATH); AwsCredentialsProvider.class, convertUtilsBean, converter, CREDENTIALS_DEFAULT_SEARCH_PATH);
this.kinesisClient = new BuilderDynaBean(KinesisAsyncClient.class, convertUtilsBean); this.kinesisClient = new BuilderDynaBean(KinesisAsyncClient.class, convertUtilsBean);
this.dynamoDbClient = new BuilderDynaBean(DynamoDbAsyncClient.class, convertUtilsBean); this.dynamoDbClient = new BuilderDynaBean(DynamoDbAsyncClient.class, convertUtilsBean);
@ -300,8 +331,8 @@ public class MultiLangDaemonConfiguration {
return credsBuilder.build(AwsCredentialsProvider.class); return credsBuilder.build(AwsCredentialsProvider.class);
} }
private void updateCredentials(BuilderDynaBean toUpdate, AwsCredentialsProvider primary, private void updateCredentials(
AwsCredentialsProvider secondary) { BuilderDynaBean toUpdate, AwsCredentialsProvider primary, AwsCredentialsProvider secondary) {
if (toUpdate.hasValue("credentialsProvider")) { if (toUpdate.hasValue("credentialsProvider")) {
return; return;
@ -329,8 +360,8 @@ public class MultiLangDaemonConfiguration {
} }
private void handleRetrievalConfig(RetrievalConfig retrievalConfig, ConfigsBuilder configsBuilder) { private void handleRetrievalConfig(RetrievalConfig retrievalConfig, ConfigsBuilder configsBuilder) {
retrievalConfig retrievalConfig.retrievalSpecificConfig(
.retrievalSpecificConfig(retrievalMode.builder(this).build(configsBuilder.kinesisClient(), this)); retrievalMode.builder(this).build(configsBuilder.kinesisClient(), this));
} }
private Object adjustKinesisHttpConfiguration(Object builderObj) { private Object adjustKinesisHttpConfiguration(Object builderObj) {
@ -353,8 +384,14 @@ public class MultiLangDaemonConfiguration {
final RetrievalConfig retrievalConfig; final RetrievalConfig retrievalConfig;
public Scheduler build() { public Scheduler build() {
return new Scheduler(checkpointConfig, coordinatorConfig, leaseManagementConfig, lifecycleConfig, return new Scheduler(
metricsConfig, processorConfig, retrievalConfig); checkpointConfig,
coordinatorConfig,
leaseManagementConfig,
lifecycleConfig,
metricsConfig,
processorConfig,
retrievalConfig);
} }
} }
@ -367,19 +404,25 @@ public class MultiLangDaemonConfiguration {
updateCredentials(dynamoDbClient, dynamoDbCreds, kinesisCreds); updateCredentials(dynamoDbClient, dynamoDbCreds, kinesisCreds);
updateCredentials(cloudWatchClient, cloudwatchCreds, kinesisCreds); updateCredentials(cloudWatchClient, cloudwatchCreds, kinesisCreds);
KinesisAsyncClient kinesisAsyncClient = kinesisClient.build(KinesisAsyncClient.class, KinesisAsyncClient kinesisAsyncClient =
this::adjustKinesisHttpConfiguration); kinesisClient.build(KinesisAsyncClient.class, this::adjustKinesisHttpConfiguration);
DynamoDbAsyncClient dynamoDbAsyncClient = dynamoDbClient.build(DynamoDbAsyncClient.class); DynamoDbAsyncClient dynamoDbAsyncClient = dynamoDbClient.build(DynamoDbAsyncClient.class);
CloudWatchAsyncClient cloudWatchAsyncClient = cloudWatchClient.build(CloudWatchAsyncClient.class); CloudWatchAsyncClient cloudWatchAsyncClient = cloudWatchClient.build(CloudWatchAsyncClient.class);
ConfigsBuilder configsBuilder = new ConfigsBuilder(streamName, applicationName, kinesisAsyncClient, ConfigsBuilder configsBuilder = new ConfigsBuilder(
dynamoDbAsyncClient, cloudWatchAsyncClient, workerIdentifier, shardRecordProcessorFactory); streamName,
applicationName,
kinesisAsyncClient,
dynamoDbAsyncClient,
cloudWatchAsyncClient,
workerIdentifier,
shardRecordProcessorFactory);
Map<Class<?>, Object> configObjects = new HashMap<>(); Map<Class<?>, Object> configObjects = new HashMap<>();
addConfigObjects(configObjects, configsBuilder); addConfigObjects(configObjects, configsBuilder);
resolveFields(configObjects, Collections.singleton(ConfigsBuilder.class), resolveFields(
Collections.singleton(PollingConfig.class)); configObjects, Collections.singleton(ConfigsBuilder.class), Collections.singleton(PollingConfig.class));
CoordinatorConfig coordinatorConfig = configsBuilder.coordinatorConfig(); CoordinatorConfig coordinatorConfig = configsBuilder.coordinatorConfig();
CheckpointConfig checkpointConfig = configsBuilder.checkpointConfig(); CheckpointConfig checkpointConfig = configsBuilder.checkpointConfig();
@ -389,19 +432,31 @@ public class MultiLangDaemonConfiguration {
ProcessorConfig processorConfig = configsBuilder.processorConfig(); ProcessorConfig processorConfig = configsBuilder.processorConfig();
RetrievalConfig retrievalConfig = configsBuilder.retrievalConfig(); RetrievalConfig retrievalConfig = configsBuilder.retrievalConfig();
addConfigObjects(configObjects, coordinatorConfig, checkpointConfig, leaseManagementConfig, lifecycleConfig, addConfigObjects(
metricsConfig, processorConfig, retrievalConfig); configObjects,
coordinatorConfig,
checkpointConfig,
leaseManagementConfig,
lifecycleConfig,
metricsConfig,
processorConfig,
retrievalConfig);
handleRetrievalConfig(retrievalConfig, configsBuilder); handleRetrievalConfig(retrievalConfig, configsBuilder);
resolveFields(configObjects, null, new HashSet<>(Arrays.asList(ConfigsBuilder.class, PollingConfig.class))); resolveFields(configObjects, null, new HashSet<>(Arrays.asList(ConfigsBuilder.class, PollingConfig.class)));
return new ResolvedConfiguration(coordinatorConfig, checkpointConfig, leaseManagementConfig, lifecycleConfig, return new ResolvedConfiguration(
metricsConfig, processorConfig, retrievalConfig); coordinatorConfig,
checkpointConfig,
leaseManagementConfig,
lifecycleConfig,
metricsConfig,
processorConfig,
retrievalConfig);
} }
public Scheduler build(ShardRecordProcessorFactory shardRecordProcessorFactory) { public Scheduler build(ShardRecordProcessorFactory shardRecordProcessorFactory) {
return resolvedConfiguration(shardRecordProcessorFactory).build(); return resolvedConfiguration(shardRecordProcessorFactory).build();
} }
}
}

View file

@ -30,34 +30,44 @@ public class PollingConfigBean implements RetrievalConfigBuilder {
interface PollingConfigBeanDelegate { interface PollingConfigBeanDelegate {
Integer getRetryGetRecordsInSeconds(); Integer getRetryGetRecordsInSeconds();
void setRetryGetRecordsInSeconds(Integer value); void setRetryGetRecordsInSeconds(Integer value);
Integer getMaxGetRecordsThreadPool(); Integer getMaxGetRecordsThreadPool();
void setMaxGetRecordsThreadPool(Integer value); void setMaxGetRecordsThreadPool(Integer value);
long getIdleTimeBetweenReadsInMillis(); long getIdleTimeBetweenReadsInMillis();
void setIdleTimeBetweenReadsInMillis(long value); void setIdleTimeBetweenReadsInMillis(long value);
int getMaxRecords(); int getMaxRecords();
void setMaxRecords(int value); void setMaxRecords(int value);
} }
@ConfigurationSettable(configurationClass = PollingConfig.class, convertToOptional = true) @ConfigurationSettable(configurationClass = PollingConfig.class, convertToOptional = true)
private Integer retryGetRecordsInSeconds; private Integer retryGetRecordsInSeconds;
@ConfigurationSettable(configurationClass = PollingConfig.class, convertToOptional = true) @ConfigurationSettable(configurationClass = PollingConfig.class, convertToOptional = true)
private Integer maxGetRecordsThreadPool; private Integer maxGetRecordsThreadPool;
@ConfigurationSettable(configurationClass = PollingConfig.class) @ConfigurationSettable(configurationClass = PollingConfig.class)
private long idleTimeBetweenReadsInMillis; private long idleTimeBetweenReadsInMillis;
@ConfigurationSettable(configurationClass = PollingConfig.class) @ConfigurationSettable(configurationClass = PollingConfig.class)
private int maxRecords; private int maxRecords;
public boolean anyPropertiesSet() { public boolean anyPropertiesSet() {
return retryGetRecordsInSeconds != null || maxGetRecordsThreadPool != null || idleTimeBetweenReadsInMillis != 0 || maxRecords != 0; return retryGetRecordsInSeconds != null
|| maxGetRecordsThreadPool != null
|| idleTimeBetweenReadsInMillis != 0
|| maxRecords != 0;
} }
@Override @Override
public PollingConfig build(KinesisAsyncClient kinesisAsyncClient, MultiLangDaemonConfiguration parent) { public PollingConfig build(KinesisAsyncClient kinesisAsyncClient, MultiLangDaemonConfiguration parent) {
return ConfigurationSettableUtils.resolveFields(this, new PollingConfig(parent.getStreamName(), kinesisAsyncClient)); return ConfigurationSettableUtils.resolveFields(
this, new PollingConfig(parent.getStreamName(), kinesisAsyncClient));
} }
} }

View file

@ -21,7 +21,7 @@ import software.amazon.kinesis.retrieval.RetrievalSpecificConfig;
public interface RetrievalConfigBuilder { public interface RetrievalConfigBuilder {
/** /**
* Creates a retrieval specific configuration using the supplied parameters, and internal class parameters * Creates a retrieval specific configuration using the supplied parameters, and internal class parameters
* *
* @param kinesisAsyncClient * @param kinesisAsyncClient
* the client that will be provided to the RetrievalSpecificConfig constructor * the client that will be provided to the RetrievalSpecificConfig constructor
* @param parent * @param parent

View file

@ -19,14 +19,14 @@ import java.util.Arrays;
import java.util.function.Function; import java.util.function.Function;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.apache.commons.lang3.Validate;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.Validate;
@Slf4j @Slf4j
public enum RetrievalMode { public enum RetrievalMode {
FANOUT(MultiLangDaemonConfiguration::getFanoutConfig), POLLING( FANOUT(MultiLangDaemonConfiguration::getFanoutConfig),
MultiLangDaemonConfiguration::getPollingConfig), DEFAULT(RetrievalMode::decideForDefault); POLLING(MultiLangDaemonConfiguration::getPollingConfig),
DEFAULT(RetrievalMode::decideForDefault);
private final Function<MultiLangDaemonConfiguration, RetrievalConfigBuilder> builderFor; private final Function<MultiLangDaemonConfiguration, RetrievalConfigBuilder> builderFor;

View file

@ -15,14 +15,13 @@
package software.amazon.kinesis.multilang.config; package software.amazon.kinesis.multilang.config;
import lombok.Data;
import java.lang.reflect.Method; import java.lang.reflect.Method;
import lombok.Data;
@Data @Data
class TypeTag { class TypeTag {
final Class<?> type; final Class<?> type;
final boolean hasConverter; final boolean hasConverter;
final Method builderMethod; final Method builderMethod;
} }

View file

@ -32,7 +32,9 @@ public class V2CredentialWrapper implements AwsCredentialsProvider {
public AwsCredentials resolveCredentials() { public AwsCredentials resolveCredentials() {
AWSCredentials current = oldCredentialsProvider.getCredentials(); AWSCredentials current = oldCredentialsProvider.getCredentials();
if (current instanceof AWSSessionCredentials) { if (current instanceof AWSSessionCredentials) {
return AwsSessionCredentials.create(current.getAWSAccessKeyId(), current.getAWSSecretKey(), return AwsSessionCredentials.create(
current.getAWSAccessKeyId(),
current.getAWSSecretKey(),
((AWSSessionCredentials) current).getSessionToken()); ((AWSSessionCredentials) current).getSessionToken());
} }
return new AwsCredentials() { return new AwsCredentials() {

View file

@ -36,6 +36,7 @@ public class CheckpointMessage extends Message {
* The checkpoint this message is about. * The checkpoint this message is about.
*/ */
private String sequenceNumber; private String sequenceNumber;
private Long subSequenceNumber; private Long subSequenceNumber;
/** /**
@ -45,7 +46,7 @@ public class CheckpointMessage extends Message {
/** /**
* Convenience constructor. * Convenience constructor.
* *
* @param sequenceNumber * @param sequenceNumber
* The sequence number that this message is about. * The sequence number that this message is about.
* @param subSequenceNumber * @param subSequenceNumber
@ -61,5 +62,4 @@ public class CheckpointMessage extends Message {
this.setError(throwable.getClass().getSimpleName()); this.setError(throwable.getClass().getSimpleName());
} }
} }
} }

View file

@ -33,29 +33,29 @@ public class InitializeMessage extends Message {
* The shard id that this processor is getting initialized for. * The shard id that this processor is getting initialized for.
*/ */
private String shardId; private String shardId;
private String sequenceNumber; private String sequenceNumber;
private Long subSequenceNumber; private Long subSequenceNumber;
/** /**
* Default constructor. * Default constructor.
*/ */
public InitializeMessage() { public InitializeMessage() {}
}
/** /**
* Convenience constructor. * Convenience constructor.
* *
* @param initializationInput {@link InitializationInput} * @param initializationInput {@link InitializationInput}
*/ */
public InitializeMessage(InitializationInput initializationInput) { public InitializeMessage(InitializationInput initializationInput) {
this.shardId = initializationInput.shardId(); this.shardId = initializationInput.shardId();
if (initializationInput.extendedSequenceNumber() != null) { if (initializationInput.extendedSequenceNumber() != null) {
this.sequenceNumber = initializationInput.extendedSequenceNumber().sequenceNumber(); this.sequenceNumber = initializationInput.extendedSequenceNumber().sequenceNumber();
this.subSequenceNumber = initializationInput.extendedSequenceNumber().subSequenceNumber(); this.subSequenceNumber =
initializationInput.extendedSequenceNumber().subSequenceNumber();
} else { } else {
this.sequenceNumber = null; this.sequenceNumber = null;
this.subSequenceNumber = null; this.subSequenceNumber = null;
} }
} }
} }

View file

@ -15,7 +15,6 @@
package software.amazon.kinesis.multilang.messages; package software.amazon.kinesis.multilang.messages;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.EqualsAndHashCode; import lombok.EqualsAndHashCode;
import lombok.Getter; import lombok.Getter;
@ -54,10 +53,11 @@ public class JsonFriendlyRecord {
data = new byte[record.data().limit()]; data = new byte[record.data().limit()];
record.data().get(data); record.data().get(data);
} }
Long approximateArrival = record.approximateArrivalTimestamp() == null ? null Long approximateArrival = record.approximateArrivalTimestamp() == null
? null
: record.approximateArrivalTimestamp().toEpochMilli(); : record.approximateArrivalTimestamp().toEpochMilli();
return new JsonFriendlyRecord(data, record.partitionKey(), record.sequenceNumber(), return new JsonFriendlyRecord(
approximateArrival, record.subSequenceNumber()); data, record.partitionKey(), record.sequenceNumber(), approximateArrival, record.subSequenceNumber());
} }
@JsonProperty @JsonProperty

View file

@ -21,5 +21,4 @@ package software.amazon.kinesis.multilang.messages;
public class LeaseLostMessage extends Message { public class LeaseLostMessage extends Message {
public static final String ACTION = "leaseLost"; public static final String ACTION = "leaseLost";
} }

View file

@ -23,15 +23,15 @@ import com.fasterxml.jackson.databind.ObjectMapper;
* Abstract class for all messages that are sent to the client's process. * Abstract class for all messages that are sent to the client's process.
*/ */
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "action") @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "action")
@JsonSubTypes({ @JsonSubTypes({
@Type(value = CheckpointMessage.class, name = CheckpointMessage.ACTION), @Type(value = CheckpointMessage.class, name = CheckpointMessage.ACTION),
@Type(value = InitializeMessage.class, name = InitializeMessage.ACTION), @Type(value = InitializeMessage.class, name = InitializeMessage.ACTION),
@Type(value = ProcessRecordsMessage.class, name = ProcessRecordsMessage.ACTION), @Type(value = ProcessRecordsMessage.class, name = ProcessRecordsMessage.ACTION),
@Type(value = ShutdownMessage.class, name = ShutdownMessage.ACTION), @Type(value = ShutdownMessage.class, name = ShutdownMessage.ACTION),
@Type(value = StatusMessage.class, name = StatusMessage.ACTION), @Type(value = StatusMessage.class, name = StatusMessage.ACTION),
@Type(value = ShutdownRequestedMessage.class, name = ShutdownRequestedMessage.ACTION), @Type(value = ShutdownRequestedMessage.class, name = ShutdownRequestedMessage.ACTION),
@Type(value = LeaseLostMessage.class, name = LeaseLostMessage.ACTION), @Type(value = LeaseLostMessage.class, name = LeaseLostMessage.ACTION),
@Type(value = ShardEndedMessage.class, name = ShardEndedMessage.ACTION), @Type(value = ShardEndedMessage.class, name = ShardEndedMessage.ACTION),
}) })
public abstract class Message { public abstract class Message {
@ -40,11 +40,10 @@ public abstract class Message {
/** /**
* Default constructor. * Default constructor.
*/ */
public Message() { public Message() {}
}
/** /**
* *
* @param objectMapper An object mapper. * @param objectMapper An object mapper.
* @return this * @return this
*/ */
@ -54,7 +53,7 @@ public abstract class Message {
} }
/** /**
* *
* @return A JSON representation of this object. * @return A JSON representation of this object.
*/ */
public String toString() { public String toString() {

View file

@ -37,17 +37,17 @@ public class ProcessRecordsMessage extends Message {
* The records that the client's process needs to handle. * The records that the client's process needs to handle.
*/ */
private List<JsonFriendlyRecord> records; private List<JsonFriendlyRecord> records;
private Long millisBehindLatest; private Long millisBehindLatest;
/** /**
* Default constructor. * Default constructor.
*/ */
public ProcessRecordsMessage() { public ProcessRecordsMessage() {}
}
/** /**
* Convenience constructor. * Convenience constructor.
* *
* @param processRecordsInput * @param processRecordsInput
* the process records input to be sent to the child * the process records input to be sent to the child
*/ */

View file

@ -142,7 +142,6 @@
* Jackson doc for more details)</a> MIME is the basis of most base64 encoding variants including <a * Jackson doc for more details)</a> MIME is the basis of most base64 encoding variants including <a
* href="http://tools.ietf.org/html/rfc3548.html">RFC 3548</a> which is the standard used by Python's <a * href="http://tools.ietf.org/html/rfc3548.html">RFC 3548</a> which is the standard used by Python's <a
* href="https://docs.python.org/2/library/base64.html">base64</a> module. * href="https://docs.python.org/2/library/base64.html">base64</a> module.
* *
*/ */
package software.amazon.kinesis.multilang; package software.amazon.kinesis.multilang;

View file

@ -14,15 +14,14 @@
*/ */
package software.amazon.kinesis.multilang; package software.amazon.kinesis.multilang;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.nullValue;
import org.hamcrest.Description; import org.hamcrest.Description;
import org.hamcrest.Matcher; import org.hamcrest.Matcher;
import org.hamcrest.TypeSafeDiagnosingMatcher; import org.hamcrest.TypeSafeDiagnosingMatcher;
import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber;
import software.amazon.kinesis.lifecycle.events.InitializationInput; import software.amazon.kinesis.lifecycle.events.InitializationInput;
import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.nullValue;
public class Matchers { public class Matchers {
@ -58,8 +57,12 @@ public class Matchers {
@Override @Override
public void describeTo(Description description) { public void describeTo(Description description) {
description.appendText("An InitializationInput matching: { shardId: ").appendDescriptionOf(shardIdMatcher) description
.appendText(", sequenceNumber: ").appendDescriptionOf(sequenceNumberMatcher).appendText(" }"); .appendText("An InitializationInput matching: { shardId: ")
.appendDescriptionOf(shardIdMatcher)
.appendText(", sequenceNumber: ")
.appendDescriptionOf(sequenceNumberMatcher)
.appendText(" }");
} }
} }
@ -98,10 +101,11 @@ public class Matchers {
@Override @Override
public void describeTo(Description description) { public void describeTo(Description description) {
description.appendText("An ExtendedSequenceNumber matching: { sequenceNumber: ") description
.appendDescriptionOf(sequenceNumberMatcher).appendText(", subSequenceNumber: ") .appendText("An ExtendedSequenceNumber matching: { sequenceNumber: ")
.appendDescriptionOf(sequenceNumberMatcher)
.appendText(", subSequenceNumber: ")
.appendDescriptionOf(subSequenceNumberMatcher); .appendDescriptionOf(subSequenceNumberMatcher);
} }
} }
} }

View file

@ -22,15 +22,14 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.Future; import java.util.concurrent.Future;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import org.mockito.Mockito; import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock; import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer; import org.mockito.stubbing.Answer;
import software.amazon.kinesis.multilang.messages.Message; import software.amazon.kinesis.multilang.messages.Message;
import software.amazon.kinesis.multilang.messages.StatusMessage; import software.amazon.kinesis.multilang.messages.StatusMessage;
import com.fasterxml.jackson.databind.ObjectMapper;
public class MessageReaderTest { public class MessageReaderTest {
@ -75,8 +74,8 @@ public class MessageReaderTest {
@Test @Test
public void runLoopGoodInputTest() { public void runLoopGoodInputTest() {
String[] sequenceNumbers = new String[] { "123", "456", "789" }; String[] sequenceNumbers = new String[] {"123", "456", "789"};
String[] responseFors = new String[] { "initialize", "processRecords", "processRecords", "shutdown" }; String[] responseFors = new String[] {"initialize", "processRecords", "processRecords", "shutdown"};
InputStream stream = buildInputStreamOfGoodInput(sequenceNumbers, responseFors); InputStream stream = buildInputStreamOfGoodInput(sequenceNumbers, responseFors);
MessageReader reader = MessageReader reader =
new MessageReader().initialize(stream, SHARD_ID, new ObjectMapper(), Executors.newCachedThreadPool()); new MessageReader().initialize(stream, SHARD_ID, new ObjectMapper(), Executors.newCachedThreadPool());
@ -85,7 +84,9 @@ public class MessageReaderTest {
try { try {
Message message = reader.getNextMessageFromSTDOUT().get(); Message message = reader.getNextMessageFromSTDOUT().get();
if (message instanceof StatusMessage) { if (message instanceof StatusMessage) {
Assert.assertEquals("The status message's responseFor field should have been correct", responseFor, Assert.assertEquals(
"The status message's responseFor field should have been correct",
responseFor,
((StatusMessage) message).getResponseFor()); ((StatusMessage) message).getResponseFor());
} }
} catch (InterruptedException | ExecutionException e) { } catch (InterruptedException | ExecutionException e) {
@ -96,8 +97,8 @@ public class MessageReaderTest {
@Test @Test
public void drainInputTest() throws InterruptedException, ExecutionException { public void drainInputTest() throws InterruptedException, ExecutionException {
String[] sequenceNumbers = new String[] { "123", "456", "789" }; String[] sequenceNumbers = new String[] {"123", "456", "789"};
String[] responseFors = new String[] { "initialize", "processRecords", "processRecords", "shutdown" }; String[] responseFors = new String[] {"initialize", "processRecords", "processRecords", "shutdown"};
InputStream stream = buildInputStreamOfGoodInput(sequenceNumbers, responseFors); InputStream stream = buildInputStreamOfGoodInput(sequenceNumbers, responseFors);
MessageReader reader = MessageReader reader =
@ -116,25 +117,26 @@ public class MessageReaderTest {
BufferedReader bufferReader = Mockito.mock(BufferedReader.class); BufferedReader bufferReader = Mockito.mock(BufferedReader.class);
try { try {
Mockito.doAnswer(new Answer() { Mockito.doAnswer(new Answer() {
private boolean returnedOnce = false; private boolean returnedOnce = false;
@Override @Override
public Object answer(InvocationOnMock invocation) throws Throwable { public Object answer(InvocationOnMock invocation) throws Throwable {
if (returnedOnce) { if (returnedOnce) {
return "{\"action\":\"status\",\"responseFor\":\"processRecords\"}"; return "{\"action\":\"status\",\"responseFor\":\"processRecords\"}";
} else { } else {
returnedOnce = true; returnedOnce = true;
return "{\"action\":\"shutdown\",\"reason\":\"ZOMBIE\"}"; return "{\"action\":\"shutdown\",\"reason\":\"ZOMBIE\"}";
} }
} }
}).when(bufferReader).readLine(); })
.when(bufferReader)
.readLine();
} catch (IOException e) { } catch (IOException e) {
Assert.fail("There shouldn't be an exception while setting up this mock."); Assert.fail("There shouldn't be an exception while setting up this mock.");
} }
MessageReader reader = MessageReader reader = new MessageReader()
new MessageReader().initialize(bufferReader, SHARD_ID, new ObjectMapper(), .initialize(bufferReader, SHARD_ID, new ObjectMapper(), Executors.newCachedThreadPool());
Executors.newCachedThreadPool());
try { try {
reader.getNextMessageFromSTDOUT().get(); reader.getNextMessageFromSTDOUT().get();
@ -165,7 +167,8 @@ public class MessageReaderTest {
readTask.get(); readTask.get();
Assert.fail("The reading task should have failed due to an IOException."); Assert.fail("The reading task should have failed due to an IOException.");
} catch (InterruptedException e) { } catch (InterruptedException e) {
Assert.fail("The reading task should not have been interrupted. It should have failed due to an IOException."); Assert.fail(
"The reading task should not have been interrupted. It should have failed due to an IOException.");
} catch (ExecutionException e) { } catch (ExecutionException e) {
// Yay!! // Yay!!
} }

View file

@ -23,21 +23,19 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.Future; import java.util.concurrent.Future;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Rule; import org.junit.Rule;
import org.junit.Test; import org.junit.Test;
import org.junit.rules.ExpectedException; import org.junit.rules.ExpectedException;
import org.mockito.Mockito; import org.mockito.Mockito;
import software.amazon.kinesis.lifecycle.events.InitializationInput;
import software.amazon.kinesis.lifecycle.events.LeaseLostInput; import software.amazon.kinesis.lifecycle.events.LeaseLostInput;
import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput;
import software.amazon.kinesis.lifecycle.events.ShardEndedInput; import software.amazon.kinesis.lifecycle.events.ShardEndedInput;
import software.amazon.kinesis.multilang.messages.Message; import software.amazon.kinesis.multilang.messages.Message;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import software.amazon.kinesis.lifecycle.events.InitializationInput;
import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput;
import software.amazon.kinesis.retrieval.KinesisClientRecord; import software.amazon.kinesis.retrieval.KinesisClientRecord;
import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verify;
@ -65,8 +63,7 @@ public class MessageWriterTest {
public void writeCheckpointMessageNoErrorTest() throws IOException, InterruptedException, ExecutionException { public void writeCheckpointMessageNoErrorTest() throws IOException, InterruptedException, ExecutionException {
Future<Boolean> future = this.messageWriter.writeCheckpointMessageWithError("1234", 0L, null); Future<Boolean> future = this.messageWriter.writeCheckpointMessageWithError("1234", 0L, null);
future.get(); future.get();
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt());
Mockito.anyInt());
verify(this.stream, Mockito.atLeastOnce()).flush(); verify(this.stream, Mockito.atLeastOnce()).flush();
} }
@ -74,42 +71,43 @@ public class MessageWriterTest {
public void writeCheckpointMessageWithErrorTest() throws IOException, InterruptedException, ExecutionException { public void writeCheckpointMessageWithErrorTest() throws IOException, InterruptedException, ExecutionException {
Future<Boolean> future = this.messageWriter.writeCheckpointMessageWithError("1234", 0L, new Throwable()); Future<Boolean> future = this.messageWriter.writeCheckpointMessageWithError("1234", 0L, new Throwable());
future.get(); future.get();
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt());
Mockito.anyInt());
verify(this.stream, Mockito.atLeastOnce()).flush(); verify(this.stream, Mockito.atLeastOnce()).flush();
} }
@Test @Test
public void writeInitializeMessageTest() throws IOException, InterruptedException, ExecutionException { public void writeInitializeMessageTest() throws IOException, InterruptedException, ExecutionException {
Future<Boolean> future = this.messageWriter.writeInitializeMessage(InitializationInput.builder().shardId(SHARD_ID).build()); Future<Boolean> future = this.messageWriter.writeInitializeMessage(
InitializationInput.builder().shardId(SHARD_ID).build());
future.get(); future.get();
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt());
Mockito.anyInt());
verify(this.stream, Mockito.atLeastOnce()).flush(); verify(this.stream, Mockito.atLeastOnce()).flush();
} }
@Test @Test
public void writeProcessRecordsMessageTest() throws IOException, InterruptedException, ExecutionException { public void writeProcessRecordsMessageTest() throws IOException, InterruptedException, ExecutionException {
List<KinesisClientRecord> records = Arrays.asList( List<KinesisClientRecord> records = Arrays.asList(
KinesisClientRecord.builder().data(ByteBuffer.wrap("kitten".getBytes())).partitionKey("some cats") KinesisClientRecord.builder()
.sequenceNumber("357234807854789057805").build(), .data(ByteBuffer.wrap("kitten".getBytes()))
KinesisClientRecord.builder().build() .partitionKey("some cats")
); .sequenceNumber("357234807854789057805")
Future<Boolean> future = this.messageWriter.writeProcessRecordsMessage(ProcessRecordsInput.builder().records(records).build()); .build(),
KinesisClientRecord.builder().build());
Future<Boolean> future = this.messageWriter.writeProcessRecordsMessage(
ProcessRecordsInput.builder().records(records).build());
future.get(); future.get();
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt());
Mockito.anyInt());
verify(this.stream, Mockito.atLeastOnce()).flush(); verify(this.stream, Mockito.atLeastOnce()).flush();
} }
@Test @Test
public void writeShutdownMessageTest() throws IOException, InterruptedException, ExecutionException { public void writeShutdownMessageTest() throws IOException, InterruptedException, ExecutionException {
Future<Boolean> future = this.messageWriter.writeShardEndedMessage(ShardEndedInput.builder().build()); Future<Boolean> future = this.messageWriter.writeShardEndedMessage(
ShardEndedInput.builder().build());
future.get(); future.get();
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt());
Mockito.anyInt());
verify(this.stream, Mockito.atLeastOnce()).flush(); verify(this.stream, Mockito.atLeastOnce()).flush();
} }
@ -118,15 +116,15 @@ public class MessageWriterTest {
Future<Boolean> future = this.messageWriter.writeShutdownRequestedMessage(); Future<Boolean> future = this.messageWriter.writeShutdownRequestedMessage();
future.get(); future.get();
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt());
Mockito.anyInt());
verify(this.stream, Mockito.atLeastOnce()).flush(); verify(this.stream, Mockito.atLeastOnce()).flush();
} }
@Test @Test
public void streamIOExceptionTest() throws IOException, InterruptedException, ExecutionException { public void streamIOExceptionTest() throws IOException, InterruptedException, ExecutionException {
Mockito.doThrow(IOException.class).when(stream).flush(); Mockito.doThrow(IOException.class).when(stream).flush();
Future<Boolean> initializeTask = this.messageWriter.writeInitializeMessage(InitializationInput.builder().shardId(SHARD_ID).build()); Future<Boolean> initializeTask = this.messageWriter.writeInitializeMessage(
InitializationInput.builder().shardId(SHARD_ID).build());
Boolean result = initializeTask.get(); Boolean result = initializeTask.get();
Assert.assertNotNull(result); Assert.assertNotNull(result);
Assert.assertFalse(result); Assert.assertFalse(result);
@ -152,7 +150,8 @@ public class MessageWriterTest {
Assert.assertFalse(this.messageWriter.isOpen()); Assert.assertFalse(this.messageWriter.isOpen());
try { try {
// Any message should fail // Any message should fail
this.messageWriter.writeInitializeMessage(InitializationInput.builder().shardId(SHARD_ID).build()); this.messageWriter.writeInitializeMessage(
InitializationInput.builder().shardId(SHARD_ID).build());
Assert.fail("MessageWriter should be closed and unable to write."); Assert.fail("MessageWriter should be closed and unable to write.");
} catch (IllegalStateException e) { } catch (IllegalStateException e) {
// This should happen. // This should happen.

View file

@ -14,26 +14,25 @@
*/ */
package software.amazon.kinesis.multilang; package software.amazon.kinesis.multilang;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.mockito.Mockito.when;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.IOException; import java.io.IOException;
import software.amazon.awssdk.regions.Region; import junit.framework.Assert;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.mockito.Mock; import org.mockito.Mock;
import org.mockito.Mockito; import org.mockito.Mockito;
import org.mockito.runners.MockitoJUnitRunner; import org.mockito.runners.MockitoJUnitRunner;
import junit.framework.Assert;
import software.amazon.awssdk.auth.credentials.AwsCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.regions.Region;
import software.amazon.kinesis.multilang.config.KinesisClientLibConfigurator; import software.amazon.kinesis.multilang.config.KinesisClientLibConfigurator;
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration; import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class) @RunWith(MockitoJUnitRunner.class)
public class MultiLangDaemonConfigTest { public class MultiLangDaemonConfigTest {
private static final String FILENAME = "multilang.properties"; private static final String FILENAME = "multilang.properties";
@ -49,6 +48,7 @@ public class MultiLangDaemonConfigTest {
@Mock @Mock
private AwsCredentialsProvider credentialsProvider; private AwsCredentialsProvider credentialsProvider;
@Mock @Mock
private AwsCredentials creds; private AwsCredentials creds;
@ -62,14 +62,13 @@ public class MultiLangDaemonConfigTest {
* @throws IOException * @throws IOException
*/ */
public void setup(String streamName, String streamArn) throws IOException { public void setup(String streamName, String streamArn) throws IOException {
String properties = String.format("executableName = %s\n" String properties = String.format(
"executableName = %s\n"
+ "applicationName = %s\n" + "applicationName = %s\n"
+ "AWSCredentialsProvider = DefaultAWSCredentialsProviderChain\n" + "AWSCredentialsProvider = DefaultAWSCredentialsProviderChain\n"
+ "processingLanguage = malbolge\n" + "processingLanguage = malbolge\n"
+ "regionName = %s\n", + "regionName = %s\n",
EXE, EXE, APPLICATION_NAME, "us-east-1");
APPLICATION_NAME,
"us-east-1");
if (streamName != null) { if (streamName != null) {
properties += String.format("streamName = %s\n", streamName); properties += String.format("streamName = %s\n", streamName);
@ -79,7 +78,8 @@ public class MultiLangDaemonConfigTest {
} }
classLoader = Mockito.mock(ClassLoader.class); classLoader = Mockito.mock(ClassLoader.class);
Mockito.doReturn(new ByteArrayInputStream(properties.getBytes())).when(classLoader) Mockito.doReturn(new ByteArrayInputStream(properties.getBytes()))
.when(classLoader)
.getResourceAsStream(FILENAME); .getResourceAsStream(FILENAME);
when(credentialsProvider.resolveCredentials()).thenReturn(creds); when(credentialsProvider.resolveCredentials()).thenReturn(creds);
@ -185,7 +185,8 @@ public class MultiLangDaemonConfigTest {
+ "AWSCredentialsProvider = DefaultAWSCredentialsProviderChain\n" + "processingLanguage = malbolge"; + "AWSCredentialsProvider = DefaultAWSCredentialsProviderChain\n" + "processingLanguage = malbolge";
ClassLoader classLoader = Mockito.mock(ClassLoader.class); ClassLoader classLoader = Mockito.mock(ClassLoader.class);
Mockito.doReturn(new ByteArrayInputStream(propertiesNoExecutableName.getBytes())).when(classLoader) Mockito.doReturn(new ByteArrayInputStream(propertiesNoExecutableName.getBytes()))
.when(classLoader)
.getResourceAsStream(FILENAME); .getResourceAsStream(FILENAME);
try { try {
@ -207,5 +208,4 @@ public class MultiLangDaemonConfigTest {
public void testActualPropertiesFile() throws Exception { public void testActualPropertiesFile() throws Exception {
new MultiLangDaemonConfig(FILENAME); new MultiLangDaemonConfig(FILENAME);
} }
}
}

View file

@ -14,6 +14,28 @@
*/ */
package software.amazon.kinesis.multilang; package software.amazon.kinesis.multilang;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.joran.JoranConfigurator;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
import org.slf4j.LoggerFactory;
import software.amazon.kinesis.coordinator.Scheduler;
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.equalTo;
@ -27,46 +49,29 @@ import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
import org.slf4j.LoggerFactory;
import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.joran.JoranConfigurator;
import software.amazon.kinesis.coordinator.Scheduler;
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
@RunWith(MockitoJUnitRunner.class) @RunWith(MockitoJUnitRunner.class)
public class MultiLangDaemonTest { public class MultiLangDaemonTest {
@Mock @Mock
private Scheduler scheduler; private Scheduler scheduler;
@Mock @Mock
private MultiLangDaemonConfig config; private MultiLangDaemonConfig config;
@Mock @Mock
private ExecutorService executorService; private ExecutorService executorService;
@Mock @Mock
private Future<Integer> futureInteger; private Future<Integer> futureInteger;
@Mock @Mock
private MultiLangDaemonConfiguration multiLangDaemonConfiguration; private MultiLangDaemonConfiguration multiLangDaemonConfiguration;
@Mock @Mock
private Runtime runtime; private Runtime runtime;
@Rule @Rule
public ExpectedException expectedException = ExpectedException.none(); public ExpectedException expectedException = ExpectedException.none();
@Rule @Rule
public final TemporaryFolder temporaryFolder = new TemporaryFolder(); public final TemporaryFolder temporaryFolder = new TemporaryFolder();
@ -86,7 +91,7 @@ public class MultiLangDaemonTest {
public void testSuccessfulNoOptionsJCommanderBuild() { public void testSuccessfulNoOptionsJCommanderBuild() {
String testPropertiesFile = "/test/properties/file"; String testPropertiesFile = "/test/properties/file";
MultiLangDaemon.MultiLangDaemonArguments arguments = new MultiLangDaemon.MultiLangDaemonArguments(); MultiLangDaemon.MultiLangDaemonArguments arguments = new MultiLangDaemon.MultiLangDaemonArguments();
daemon.buildJCommanderAndParseArgs(arguments, new String[] { testPropertiesFile }); daemon.buildJCommanderAndParseArgs(arguments, new String[] {testPropertiesFile});
assertThat(arguments.propertiesFile, nullValue()); assertThat(arguments.propertiesFile, nullValue());
assertThat(arguments.logConfiguration, nullValue()); assertThat(arguments.logConfiguration, nullValue());
@ -98,7 +103,7 @@ public class MultiLangDaemonTest {
public void testSuccessfulOptionsJCommanderBuild() { public void testSuccessfulOptionsJCommanderBuild() {
String propertiesOption = "/test/properties/file/option"; String propertiesOption = "/test/properties/file/option";
String propertiesFileArgs = "/test/properties/args"; String propertiesFileArgs = "/test/properties/args";
String[] args = new String[] { "-p", propertiesOption, propertiesFileArgs }; String[] args = new String[] {"-p", propertiesOption, propertiesFileArgs};
MultiLangDaemon.MultiLangDaemonArguments arguments = new MultiLangDaemon.MultiLangDaemonArguments(); MultiLangDaemon.MultiLangDaemonArguments arguments = new MultiLangDaemon.MultiLangDaemonArguments();
daemon.buildJCommanderAndParseArgs(arguments, args); daemon.buildJCommanderAndParseArgs(arguments, args);
@ -124,7 +129,8 @@ public class MultiLangDaemonTest {
LoggerContext loggerContext = spy((LoggerContext) LoggerFactory.getILoggerFactory()); LoggerContext loggerContext = spy((LoggerContext) LoggerFactory.getILoggerFactory());
JoranConfigurator configurator = spy(new JoranConfigurator()); JoranConfigurator configurator = spy(new JoranConfigurator());
String logConfiguration = this.getClass().getClassLoader().getResource("logback.xml").getPath(); String logConfiguration =
this.getClass().getClassLoader().getResource("logback.xml").getPath();
daemon.configureLogging(logConfiguration, loggerContext, configurator); daemon.configureLogging(logConfiguration, loggerContext, configurator);
verify(loggerContext).reset(); verify(loggerContext).reset();

View file

@ -14,6 +14,42 @@
*/ */
package software.amazon.kinesis.multilang; package software.amazon.kinesis.multilang;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import com.google.common.util.concurrent.SettableFuture;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.runners.MockitoJUnitRunner;
import org.mockito.stubbing.Answer;
import software.amazon.kinesis.exceptions.InvalidStateException;
import software.amazon.kinesis.exceptions.KinesisClientLibDependencyException;
import software.amazon.kinesis.exceptions.ShutdownException;
import software.amazon.kinesis.exceptions.ThrottlingException;
import software.amazon.kinesis.lifecycle.events.InitializationInput;
import software.amazon.kinesis.lifecycle.events.LeaseLostInput;
import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput;
import software.amazon.kinesis.lifecycle.events.ShardEndedInput;
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
import software.amazon.kinesis.multilang.messages.CheckpointMessage;
import software.amazon.kinesis.multilang.messages.LeaseLostMessage;
import software.amazon.kinesis.multilang.messages.Message;
import software.amazon.kinesis.multilang.messages.ProcessRecordsMessage;
import software.amazon.kinesis.multilang.messages.ShardEndedMessage;
import software.amazon.kinesis.multilang.messages.StatusMessage;
import software.amazon.kinesis.processor.RecordProcessorCheckpointer;
import software.amazon.kinesis.retrieval.KinesisClientRecord;
import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.equalTo;
import static org.junit.Assert.assertThat; import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
@ -27,65 +63,35 @@ import static org.mockito.Mockito.timeout;
import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.runners.MockitoJUnitRunner;
import org.mockito.stubbing.Answer;
import software.amazon.kinesis.exceptions.InvalidStateException;
import software.amazon.kinesis.exceptions.KinesisClientLibDependencyException;
import software.amazon.kinesis.exceptions.ShutdownException;
import software.amazon.kinesis.exceptions.ThrottlingException;
import software.amazon.kinesis.lifecycle.events.LeaseLostInput;
import software.amazon.kinesis.lifecycle.events.ShardEndedInput;
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
import software.amazon.kinesis.multilang.messages.CheckpointMessage;
import software.amazon.kinesis.multilang.messages.LeaseLostMessage;
import software.amazon.kinesis.multilang.messages.Message;
import software.amazon.kinesis.multilang.messages.ProcessRecordsMessage;
import software.amazon.kinesis.multilang.messages.ShardEndedMessage;
import software.amazon.kinesis.multilang.messages.StatusMessage;
import com.google.common.util.concurrent.SettableFuture;
import software.amazon.kinesis.lifecycle.events.InitializationInput;
import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput;
import software.amazon.kinesis.processor.RecordProcessorCheckpointer;
import software.amazon.kinesis.retrieval.KinesisClientRecord;
@RunWith(MockitoJUnitRunner.class) @RunWith(MockitoJUnitRunner.class)
public class MultiLangProtocolTest { public class MultiLangProtocolTest {
private static final List<KinesisClientRecord> EMPTY_RECORD_LIST = Collections.emptyList(); private static final List<KinesisClientRecord> EMPTY_RECORD_LIST = Collections.emptyList();
@Mock @Mock
private MultiLangProtocol protocol; private MultiLangProtocol protocol;
@Mock @Mock
private MessageWriter messageWriter; private MessageWriter messageWriter;
@Mock @Mock
private MessageReader messageReader; private MessageReader messageReader;
private String shardId; private String shardId;
@Mock @Mock
private RecordProcessorCheckpointer checkpointer; private RecordProcessorCheckpointer checkpointer;
@Mock @Mock
private MultiLangDaemonConfiguration configuration; private MultiLangDaemonConfiguration configuration;
@Before @Before
public void setup() { public void setup() {
this.shardId = "shard-id-123"; this.shardId = "shard-id-123";
protocol = new MultiLangProtocolForTesting(messageReader, messageWriter, protocol = new MultiLangProtocolForTesting(
InitializationInput.builder().shardId(shardId).build(), configuration); messageReader,
messageWriter,
InitializationInput.builder().shardId(shardId).build(),
configuration);
when(configuration.getTimeoutInSeconds()).thenReturn(null); when(configuration.getTimeoutInSeconds()).thenReturn(null);
} }
@ -104,28 +110,32 @@ public class MultiLangProtocolTest {
@Test @Test
public void testInitialize() { public void testInitialize() {
when(messageWriter when(messageWriter.writeInitializeMessage(argThat(Matchers.withInit(
.writeInitializeMessage(argThat(Matchers.withInit(InitializationInput.builder() InitializationInput.builder().shardId(shardId).build()))))
.shardId(shardId).build())))).thenReturn(buildFuture(true)); .thenReturn(buildFuture(true));
when(messageReader.getNextMessageFromSTDOUT()).thenReturn(buildFuture( when(messageReader.getNextMessageFromSTDOUT())
new StatusMessage("initialize"), Message.class)); .thenReturn(buildFuture(new StatusMessage("initialize"), Message.class));
assertThat(protocol.initialize(), equalTo(true)); assertThat(protocol.initialize(), equalTo(true));
} }
@Test @Test
public void testProcessRecords() { public void testProcessRecords() {
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class))).thenReturn(buildFuture(true)); when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class)))
when(messageReader.getNextMessageFromSTDOUT()).thenReturn(buildFuture( .thenReturn(buildFuture(true));
new StatusMessage("processRecords"), Message.class)); when(messageReader.getNextMessageFromSTDOUT())
.thenReturn(buildFuture(new StatusMessage("processRecords"), Message.class));
assertThat(protocol.processRecords(ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST).build()), assertThat(
protocol.processRecords(
ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST).build()),
equalTo(true)); equalTo(true));
} }
@Test @Test
public void leaseLostTest() { public void leaseLostTest() {
when(messageWriter.writeLeaseLossMessage(any(LeaseLostInput.class))).thenReturn(buildFuture(true)); when(messageWriter.writeLeaseLossMessage(any(LeaseLostInput.class))).thenReturn(buildFuture(true));
when(messageReader.getNextMessageFromSTDOUT()).thenReturn(buildFuture(new StatusMessage(LeaseLostMessage.ACTION), Message.class)); when(messageReader.getNextMessageFromSTDOUT())
.thenReturn(buildFuture(new StatusMessage(LeaseLostMessage.ACTION), Message.class));
assertThat(protocol.leaseLost(LeaseLostInput.builder().build()), equalTo(true)); assertThat(protocol.leaseLost(LeaseLostInput.builder().build()), equalTo(true));
} }
@ -133,7 +143,8 @@ public class MultiLangProtocolTest {
@Test @Test
public void shardEndedTest() { public void shardEndedTest() {
when(messageWriter.writeShardEndedMessage(any(ShardEndedInput.class))).thenReturn(buildFuture(true)); when(messageWriter.writeShardEndedMessage(any(ShardEndedInput.class))).thenReturn(buildFuture(true));
when(messageReader.getNextMessageFromSTDOUT()).thenReturn(buildFuture(new StatusMessage(ShardEndedMessage.ACTION))); when(messageReader.getNextMessageFromSTDOUT())
.thenReturn(buildFuture(new StatusMessage(ShardEndedMessage.ACTION)));
assertThat(protocol.shardEnded(ShardEndedInput.builder().build()), equalTo(true)); assertThat(protocol.shardEnded(ShardEndedInput.builder().build()), equalTo(true));
} }
@ -141,12 +152,12 @@ public class MultiLangProtocolTest {
@Test @Test
public void shutdownRequestedTest() { public void shutdownRequestedTest() {
when(messageWriter.writeShutdownRequestedMessage()).thenReturn(buildFuture(true)); when(messageWriter.writeShutdownRequestedMessage()).thenReturn(buildFuture(true));
when(messageReader.getNextMessageFromSTDOUT()).thenReturn(buildFuture( when(messageReader.getNextMessageFromSTDOUT())
new StatusMessage("shutdownRequested"), Message.class)); .thenReturn(buildFuture(new StatusMessage("shutdownRequested"), Message.class));
Mockito.doReturn(buildFuture(true)).when(messageWriter) Mockito.doReturn(buildFuture(true)).when(messageWriter).writeShutdownRequestedMessage();
.writeShutdownRequestedMessage();
Mockito.doReturn(buildFuture(new StatusMessage("shutdownRequested"))) Mockito.doReturn(buildFuture(new StatusMessage("shutdownRequested")))
.when(messageReader).getNextMessageFromSTDOUT(); .when(messageReader)
.getNextMessageFromSTDOUT();
assertThat(protocol.shutdownRequested(null), equalTo(true)); assertThat(protocol.shutdownRequested(null), equalTo(true));
} }
@ -168,16 +179,17 @@ public class MultiLangProtocolTest {
} }
return buildFuture(message); return buildFuture(message);
} }
}.init(messages); }.init(messages);
} }
@Test @Test
public void testProcessRecordsWithCheckpoints() throws public void testProcessRecordsWithCheckpoints()
KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException { throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class))).thenReturn(buildFuture(true)); when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class)))
when(messageWriter.writeCheckpointMessageWithError(anyString(), anyLong(), any(Throwable.class))).thenReturn(buildFuture(true)); .thenReturn(buildFuture(true));
when(messageWriter.writeCheckpointMessageWithError(anyString(), anyLong(), any(Throwable.class)))
.thenReturn(buildFuture(true));
when(messageReader.getNextMessageFromSTDOUT()).thenAnswer(buildMessageAnswers(new ArrayList<Message>() { when(messageReader.getNextMessageFromSTDOUT()).thenAnswer(buildMessageAnswers(new ArrayList<Message>() {
{ {
this.add(new CheckpointMessage("123", 0L, null)); this.add(new CheckpointMessage("123", 0L, null));
@ -192,8 +204,10 @@ public class MultiLangProtocolTest {
} }
})); }));
boolean result = protocol.processRecords(ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST) boolean result = protocol.processRecords(ProcessRecordsInput.builder()
.checkpointer(checkpointer).build()); .records(EMPTY_RECORD_LIST)
.checkpointer(checkpointer)
.build());
assertThat(result, equalTo(true)); assertThat(result, equalTo(true));
@ -203,41 +217,52 @@ public class MultiLangProtocolTest {
@Test @Test
public void testProcessRecordsWithABadCheckpoint() { public void testProcessRecordsWithABadCheckpoint() {
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class))).thenReturn(buildFuture(true)); when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class)))
when(messageWriter.writeCheckpointMessageWithError(anyString(), anyLong(), any(Throwable.class))).thenReturn(buildFuture(false)); .thenReturn(buildFuture(true));
when(messageWriter.writeCheckpointMessageWithError(anyString(), anyLong(), any(Throwable.class)))
.thenReturn(buildFuture(false));
when(messageReader.getNextMessageFromSTDOUT()).thenAnswer(buildMessageAnswers(new ArrayList<Message>() { when(messageReader.getNextMessageFromSTDOUT()).thenAnswer(buildMessageAnswers(new ArrayList<Message>() {
{ {
this.add(new CheckpointMessage("456", 0L, null)); this.add(new CheckpointMessage("456", 0L, null));
this.add(new StatusMessage("processRecords")); this.add(new StatusMessage("processRecords"));
} }
})); }));
assertThat(protocol.processRecords(ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST) assertThat(
.checkpointer(checkpointer).build()), equalTo(false)); protocol.processRecords(ProcessRecordsInput.builder()
.records(EMPTY_RECORD_LIST)
.checkpointer(checkpointer)
.build()),
equalTo(false));
} }
@Test(expected = NullPointerException.class) @Test(expected = NullPointerException.class)
public void waitForStatusMessageTimeoutTest() throws InterruptedException, TimeoutException, ExecutionException { public void waitForStatusMessageTimeoutTest() throws InterruptedException, TimeoutException, ExecutionException {
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class))).thenReturn(buildFuture(true)); when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class)))
.thenReturn(buildFuture(true));
Future<Message> future = Mockito.mock(Future.class); Future<Message> future = Mockito.mock(Future.class);
when(messageReader.getNextMessageFromSTDOUT()).thenReturn(future); when(messageReader.getNextMessageFromSTDOUT()).thenReturn(future);
when(configuration.getTimeoutInSeconds()).thenReturn(5); when(configuration.getTimeoutInSeconds()).thenReturn(5);
when(future.get(anyInt(), eq(TimeUnit.SECONDS))).thenThrow(TimeoutException.class); when(future.get(anyInt(), eq(TimeUnit.SECONDS))).thenThrow(TimeoutException.class);
protocol = new MultiLangProtocolForTesting(messageReader, protocol = new MultiLangProtocolForTesting(
messageReader,
messageWriter, messageWriter,
InitializationInput.builder().shardId(shardId).build(), InitializationInput.builder().shardId(shardId).build(),
configuration); configuration);
protocol.processRecords(ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST).build()); protocol.processRecords(
ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST).build());
} }
@Test @Test
public void waitForStatusMessageSuccessTest() { public void waitForStatusMessageSuccessTest() {
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class))).thenReturn(buildFuture(true)); when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class)))
when(messageReader.getNextMessageFromSTDOUT()).thenReturn(buildFuture( .thenReturn(buildFuture(true));
new StatusMessage("processRecords"), Message.class)); when(messageReader.getNextMessageFromSTDOUT())
.thenReturn(buildFuture(new StatusMessage("processRecords"), Message.class));
when(configuration.getTimeoutInSeconds()).thenReturn(5); when(configuration.getTimeoutInSeconds()).thenReturn(5);
assertTrue(protocol.processRecords(ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST).build())); assertTrue(protocol.processRecords(
ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST).build()));
} }
private class MultiLangProtocolForTesting extends MultiLangProtocol { private class MultiLangProtocolForTesting extends MultiLangProtocol {
@ -249,10 +274,11 @@ public class MultiLangProtocolTest {
* @param initializationInput * @param initializationInput
* @param configuration * @param configuration
*/ */
MultiLangProtocolForTesting(final MessageReader messageReader, MultiLangProtocolForTesting(
final MessageWriter messageWriter, final MessageReader messageReader,
final InitializationInput initializationInput, final MessageWriter messageWriter,
final MultiLangDaemonConfiguration configuration) { final InitializationInput initializationInput,
final MultiLangDaemonConfiguration configuration) {
super(messageReader, messageWriter, initializationInput, configuration); super(messageReader, messageWriter, initializationInput, configuration);
} }

View file

@ -15,6 +15,10 @@
package software.amazon.kinesis.multilang; package software.amazon.kinesis.multilang;
import com.amazonaws.regions.Regions; import com.amazonaws.regions.Regions;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verify;
@ -24,11 +28,6 @@ import static software.amazon.kinesis.multilang.NestedPropertyKey.ENDPOINT_REGIO
import static software.amazon.kinesis.multilang.NestedPropertyKey.EXTERNAL_ID; import static software.amazon.kinesis.multilang.NestedPropertyKey.EXTERNAL_ID;
import static software.amazon.kinesis.multilang.NestedPropertyKey.parse; import static software.amazon.kinesis.multilang.NestedPropertyKey.parse;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
@RunWith(MockitoJUnitRunner.class) @RunWith(MockitoJUnitRunner.class)
public class NestedPropertyKeyTest { public class NestedPropertyKeyTest {
@ -95,11 +94,11 @@ public class NestedPropertyKeyTest {
@Test @Test
public void testNonmatchingParameters() { public void testNonmatchingParameters() {
final String[] params = new String[] { final String[] params = new String[] {
null, null,
"", "",
"hello world", // no nested key "hello world", // no nested key
"foo=bar", // nested key, but is not a recognized key "foo=bar", // nested key, but is not a recognized key
createKey(EXTERNAL_ID, "eid") + "=extra", // valid key made invalid by second '=' createKey(EXTERNAL_ID, "eid") + "=extra", // valid key made invalid by second '='
}; };
parse(mockProcessor, params); parse(mockProcessor, params);
verifyZeroInteractions(mockProcessor); verifyZeroInteractions(mockProcessor);
@ -108,5 +107,4 @@ public class NestedPropertyKeyTest {
private static String createKey(final NestedPropertyKey key, final String value) { private static String createKey(final NestedPropertyKey key, final String value) {
return key.getNestedKey() + "=" + value; return key.getNestedKey() + "=" + value;
} }
}
}

View file

@ -72,7 +72,8 @@ public class ReadSTDERRTaskTest {
try { try {
finishedCleanly = result.get(); finishedCleanly = result.get();
} catch (InterruptedException | ExecutionException e) { } catch (InterruptedException | ExecutionException e) {
Assert.fail("Should have been able to get a result. The error should be handled during the call and result in false."); Assert.fail(
"Should have been able to get a result. The error should be handled during the call and result in false.");
} }
Assert.assertFalse("Reading a line should have thrown an exception", finishedCleanly); Assert.assertFalse("Reading a line should have thrown an exception", finishedCleanly);
} }

View file

@ -16,12 +16,11 @@ package software.amazon.kinesis.multilang;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
import software.amazon.kinesis.processor.ShardRecordProcessor;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.mockito.Mock; import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner; import org.mockito.runners.MockitoJUnitRunner;
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
import software.amazon.kinesis.processor.ShardRecordProcessor;
@RunWith(MockitoJUnitRunner.class) @RunWith(MockitoJUnitRunner.class)
public class StreamingShardRecordProcessorFactoryTest { public class StreamingShardRecordProcessorFactoryTest {
@ -31,10 +30,13 @@ public class StreamingShardRecordProcessorFactoryTest {
@Test @Test
public void createProcessorTest() { public void createProcessorTest() {
MultiLangRecordProcessorFactory factory = new MultiLangRecordProcessorFactory("somecommand", null, configuration); MultiLangRecordProcessorFactory factory =
new MultiLangRecordProcessorFactory("somecommand", null, configuration);
ShardRecordProcessor processor = factory.shardRecordProcessor(); ShardRecordProcessor processor = factory.shardRecordProcessor();
Assert.assertEquals("Should have constructed a StreamingRecordProcessor", MultiLangShardRecordProcessor.class, Assert.assertEquals(
"Should have constructed a StreamingRecordProcessor",
MultiLangShardRecordProcessor.class,
processor.getClass()); processor.getClass());
} }
} }

View file

@ -14,15 +14,6 @@
*/ */
package software.amazon.kinesis.multilang; package software.amazon.kinesis.multilang;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Matchers.anyString;
import static org.mockito.Matchers.argThat;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.InputStream; import java.io.InputStream;
import java.io.OutputStream; import java.io.OutputStream;
import java.util.Collections; import java.util.Collections;
@ -32,6 +23,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.Future; import java.util.concurrent.Future;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@ -41,9 +33,6 @@ import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock; import org.mockito.invocation.InvocationOnMock;
import org.mockito.runners.MockitoJUnitRunner; import org.mockito.runners.MockitoJUnitRunner;
import org.mockito.stubbing.Answer; import org.mockito.stubbing.Answer;
import com.fasterxml.jackson.databind.ObjectMapper;
import software.amazon.awssdk.services.kinesis.model.Record; import software.amazon.awssdk.services.kinesis.model.Record;
import software.amazon.kinesis.exceptions.KinesisClientLibDependencyException; import software.amazon.kinesis.exceptions.KinesisClientLibDependencyException;
import software.amazon.kinesis.exceptions.ThrottlingException; import software.amazon.kinesis.exceptions.ThrottlingException;
@ -61,6 +50,15 @@ import software.amazon.kinesis.processor.PreparedCheckpointer;
import software.amazon.kinesis.processor.RecordProcessorCheckpointer; import software.amazon.kinesis.processor.RecordProcessorCheckpointer;
import software.amazon.kinesis.retrieval.KinesisClientRecord; import software.amazon.kinesis.retrieval.KinesisClientRecord;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Matchers.anyString;
import static org.mockito.Matchers.argThat;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class) @RunWith(MockitoJUnitRunner.class)
public class StreamingShardRecordProcessorTest { public class StreamingShardRecordProcessorTest {
@ -70,6 +68,7 @@ public class StreamingShardRecordProcessorTest {
@Mock @Mock
private Future<Message> messageFuture; private Future<Message> messageFuture;
@Mock @Mock
private Future<Boolean> trueFuture; private Future<Boolean> trueFuture;
@ -81,14 +80,13 @@ public class StreamingShardRecordProcessorTest {
} }
@Override @Override
public void checkpoint(String sequenceNumber) throws KinesisClientLibDependencyException, public void checkpoint(String sequenceNumber)
ThrottlingException, IllegalArgumentException { throws KinesisClientLibDependencyException, ThrottlingException, IllegalArgumentException {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
@Override @Override
public void checkpoint(Record record) public void checkpoint(Record record) throws KinesisClientLibDependencyException, ThrottlingException {
throws KinesisClientLibDependencyException, ThrottlingException {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
@ -141,7 +139,8 @@ public class StreamingShardRecordProcessorTest {
} }
@Override @Override
public PreparedCheckpointer prepareCheckpoint(String sequenceNumber, long subSequenceNumber, byte[] applicationState) public PreparedCheckpointer prepareCheckpoint(
String sequenceNumber, long subSequenceNumber, byte[] applicationState)
throws KinesisClientLibDependencyException, ThrottlingException, IllegalArgumentException { throws KinesisClientLibDependencyException, ThrottlingException, IllegalArgumentException {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
@ -178,8 +177,14 @@ public class StreamingShardRecordProcessorTest {
when(configuration.getTimeoutInSeconds()).thenReturn(null); when(configuration.getTimeoutInSeconds()).thenReturn(null);
recordProcessor = recordProcessor =
new MultiLangShardRecordProcessor(new ProcessBuilder(), executor, new ObjectMapper(), messageWriter, new MultiLangShardRecordProcessor(
messageReader, errorReader, configuration) { new ProcessBuilder(),
executor,
new ObjectMapper(),
messageWriter,
messageReader,
errorReader,
configuration) {
// Just don't do anything when we exit. // Just don't do anything when we exit.
void exit() { void exit() {
@ -203,9 +208,12 @@ public class StreamingShardRecordProcessorTest {
Mockito.doReturn(Mockito.mock(Future.class)).when(messageReader).drainSTDOUT(); Mockito.doReturn(Mockito.mock(Future.class)).when(messageReader).drainSTDOUT();
Mockito.doReturn(true).when(trueFuture).get(); Mockito.doReturn(true).when(trueFuture).get();
when(messageWriter.writeInitializeMessage(any(InitializationInput.class))).thenReturn(trueFuture); when(messageWriter.writeInitializeMessage(any(InitializationInput.class)))
when(messageWriter.writeCheckpointMessageWithError(anyString(), anyLong(), any(Throwable.class))).thenReturn(trueFuture); .thenReturn(trueFuture);
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class))).thenReturn(trueFuture); when(messageWriter.writeCheckpointMessageWithError(anyString(), anyLong(), any(Throwable.class)))
.thenReturn(trueFuture);
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class)))
.thenReturn(trueFuture);
when(messageWriter.writeLeaseLossMessage(any(LeaseLostInput.class))).thenReturn(trueFuture); when(messageWriter.writeLeaseLossMessage(any(LeaseLostInput.class))).thenReturn(trueFuture);
} }
@ -223,11 +231,16 @@ public class StreamingShardRecordProcessorTest {
List<KinesisClientRecord> testRecords = Collections.emptyList(); List<KinesisClientRecord> testRecords = Collections.emptyList();
recordProcessor.initialize(InitializationInput.builder().shardId(SHARD_ID).build()); recordProcessor.initialize(
recordProcessor.processRecords(ProcessRecordsInput.builder().records(testRecords) InitializationInput.builder().shardId(SHARD_ID).build());
.checkpointer(unimplementedCheckpointer).build()); recordProcessor.processRecords(ProcessRecordsInput.builder()
recordProcessor.processRecords(ProcessRecordsInput.builder().records(testRecords) .records(testRecords)
.checkpointer(unimplementedCheckpointer).build()); .checkpointer(unimplementedCheckpointer)
.build());
recordProcessor.processRecords(ProcessRecordsInput.builder()
.records(testRecords)
.checkpointer(unimplementedCheckpointer)
.build());
recordProcessor.leaseLost(LeaseLostInput.builder().build()); recordProcessor.leaseLost(LeaseLostInput.builder().build());
} }
@ -235,9 +248,12 @@ public class StreamingShardRecordProcessorTest {
public void processorPhasesTest() throws InterruptedException, ExecutionException { public void processorPhasesTest() throws InterruptedException, ExecutionException {
Answer<StatusMessage> answer = new Answer<StatusMessage>() { Answer<StatusMessage> answer = new Answer<StatusMessage>() {
StatusMessage[] answers = new StatusMessage[] { new StatusMessage(InitializeMessage.ACTION), StatusMessage[] answers = new StatusMessage[] {
new StatusMessage(ProcessRecordsMessage.ACTION), new StatusMessage(ProcessRecordsMessage.ACTION), new StatusMessage(InitializeMessage.ACTION),
new StatusMessage(ShutdownMessage.ACTION) }; new StatusMessage(ProcessRecordsMessage.ACTION),
new StatusMessage(ProcessRecordsMessage.ACTION),
new StatusMessage(ShutdownMessage.ACTION)
};
int callCount = 0; int callCount = 0;
@ -268,9 +284,12 @@ public class StreamingShardRecordProcessorTest {
* This bad message will cause shutdown to not attempt to send a message. i.e. avoid encountering an * This bad message will cause shutdown to not attempt to send a message. i.e. avoid encountering an
* exception. * exception.
*/ */
StatusMessage[] answers = new StatusMessage[] { new StatusMessage("Bad"), StatusMessage[] answers = new StatusMessage[] {
new StatusMessage(ProcessRecordsMessage.ACTION), new StatusMessage(ProcessRecordsMessage.ACTION), new StatusMessage("Bad"),
new StatusMessage(ShutdownMessage.ACTION) }; new StatusMessage(ProcessRecordsMessage.ACTION),
new StatusMessage(ProcessRecordsMessage.ACTION),
new StatusMessage(ShutdownMessage.ACTION)
};
int callCount = 0; int callCount = 0;
@ -286,8 +305,9 @@ public class StreamingShardRecordProcessorTest {
phases(answer); phases(answer);
verify(messageWriter).writeInitializeMessage(argThat(Matchers.withInit(InitializationInput.builder() verify(messageWriter)
.shardId(SHARD_ID).build()))); .writeInitializeMessage(argThat(Matchers.withInit(
InitializationInput.builder().shardId(SHARD_ID).build())));
verify(messageWriter, times(2)).writeProcessRecordsMessage(any(ProcessRecordsInput.class)); verify(messageWriter, times(2)).writeProcessRecordsMessage(any(ProcessRecordsInput.class));
verify(messageWriter, never()).writeLeaseLossMessage(any(LeaseLostInput.class)); verify(messageWriter, never()).writeLeaseLossMessage(any(LeaseLostInput.class));
Assert.assertEquals(1, systemExitCount); Assert.assertEquals(1, systemExitCount);

View file

@ -16,10 +16,10 @@ package software.amazon.kinesis.multilang.auth;
import java.util.Arrays; import java.util.Arrays;
import static org.junit.Assert.assertEquals;
import org.junit.Test; import org.junit.Test;
import static org.junit.Assert.assertEquals;
public class KclSTSAssumeRoleSessionCredentialsProviderTest { public class KclSTSAssumeRoleSessionCredentialsProviderTest {
private static final String ARN = "arn"; private static final String ARN = "arn";
@ -31,7 +31,7 @@ public class KclSTSAssumeRoleSessionCredentialsProviderTest {
*/ */
@Test @Test
public void testConstructorWithoutOptionalParams() { public void testConstructorWithoutOptionalParams() {
new KclSTSAssumeRoleSessionCredentialsProvider(new String[] { ARN, SESSION_NAME }); new KclSTSAssumeRoleSessionCredentialsProvider(new String[] {ARN, SESSION_NAME});
} }
@Test @Test
@ -46,9 +46,8 @@ public class KclSTSAssumeRoleSessionCredentialsProviderTest {
@Test @Test
public void testVarArgs() { public void testVarArgs() {
for (final String[] varargs : Arrays.asList( for (final String[] varargs : Arrays.asList(
new String[] { ARN, SESSION_NAME, "externalId=eid", "foo"}, new String[] {ARN, SESSION_NAME, "externalId=eid", "foo"},
new String[] { ARN, SESSION_NAME, "foo", "externalId=eid"} new String[] {ARN, SESSION_NAME, "foo", "externalId=eid"})) {
)) {
final VarArgsSpy provider = new VarArgsSpy(varargs); final VarArgsSpy provider = new VarArgsSpy(varargs);
assertEquals("eid", provider.externalId); assertEquals("eid", provider.externalId);
} }
@ -68,4 +67,4 @@ public class KclSTSAssumeRoleSessionCredentialsProviderTest {
super.acceptExternalId(externalId); super.acceptExternalId(externalId);
} }
} }
} }

View file

@ -14,14 +14,11 @@
*/ */
package software.amazon.kinesis.multilang.config; package software.amazon.kinesis.multilang.config;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
import java.util.Arrays; import java.util.Arrays;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSCredentialsProviderChain;
import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.auth.BasicAWSCredentials;
import lombok.ToString; import lombok.ToString;
import org.hamcrest.Description; import org.hamcrest.Description;
@ -30,9 +27,11 @@ import org.hamcrest.TypeSafeDiagnosingMatcher;
import org.junit.Test; import org.junit.Test;
import software.amazon.kinesis.multilang.auth.KclSTSAssumeRoleSessionCredentialsProvider; import software.amazon.kinesis.multilang.auth.KclSTSAssumeRoleSessionCredentialsProvider;
import com.amazonaws.auth.AWSCredentials; import static org.hamcrest.CoreMatchers.equalTo;
import com.amazonaws.auth.AWSCredentialsProvider; import static org.hamcrest.CoreMatchers.instanceOf;
import com.amazonaws.auth.AWSCredentialsProviderChain; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
public class AWSCredentialsProviderPropertyValueDecoderTest { public class AWSCredentialsProviderPropertyValueDecoderTest {
@ -79,10 +78,10 @@ public class AWSCredentialsProviderPropertyValueDecoderTest {
@Override @Override
public void describeTo(Description description) { public void describeTo(Description description) {
description.appendText("An AWSCredentialsProvider that provides an AWSCredential matching: ") description
.appendText("An AWSCredentialsProvider that provides an AWSCredential matching: ")
.appendList("(", ", ", ")", Arrays.asList(classMatcher, akidMatcher, secretMatcher)); .appendList("(", ", ", ")", Arrays.asList(classMatcher, akidMatcher, secretMatcher));
} }
} }
private static AWSCredentialsMatcher hasCredentials(String akid, String secret) { private static AWSCredentialsMatcher hasCredentials(String akid, String secret) {
@ -121,7 +120,7 @@ public class AWSCredentialsProviderPropertyValueDecoderTest {
for (final String className : Arrays.asList( for (final String className : Arrays.asList(
KclSTSAssumeRoleSessionCredentialsProvider.class.getName(), // fully-qualified name KclSTSAssumeRoleSessionCredentialsProvider.class.getName(), // fully-qualified name
KclSTSAssumeRoleSessionCredentialsProvider.class.getSimpleName() // name-only; needs prefix KclSTSAssumeRoleSessionCredentialsProvider.class.getSimpleName() // name-only; needs prefix
)) { )) {
final AWSCredentialsProvider provider = decoder.decodeValue(className + "|arn|sessionName"); final AWSCredentialsProvider provider = decoder.decodeValue(className + "|arn|sessionName");
assertNotNull(className, provider); assertNotNull(className, provider);
} }
@ -132,7 +131,7 @@ public class AWSCredentialsProviderPropertyValueDecoderTest {
*/ */
@Test @Test
public void testVarArgAuthProvider() { public void testVarArgAuthProvider() {
final String[] args = new String[] { "arg1", "arg2", "arg3" }; final String[] args = new String[] {"arg1", "arg2", "arg3"};
final String className = VarArgCredentialsProvider.class.getName(); final String className = VarArgCredentialsProvider.class.getName();
final String encodedValue = className + "|" + String.join("|", args); final String encodedValue = className + "|" + String.join("|", args);
@ -151,9 +150,7 @@ public class AWSCredentialsProviderPropertyValueDecoderTest {
} }
@Override @Override
public void refresh() { public void refresh() {}
}
} }
/** /**
@ -180,9 +177,7 @@ public class AWSCredentialsProviderPropertyValueDecoderTest {
} }
@Override @Override
public void refresh() { public void refresh() {}
}
} }
private static class VarArgCredentialsProvider implements AWSCredentialsProvider { private static class VarArgCredentialsProvider implements AWSCredentialsProvider {
@ -201,8 +196,6 @@ public class AWSCredentialsProviderPropertyValueDecoderTest {
} }
@Override @Override
public void refresh() { public void refresh() {}
}
} }
} }

View file

@ -15,16 +15,14 @@
package software.amazon.kinesis.multilang.config; package software.amazon.kinesis.multilang.config;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.CoreMatchers.nullValue;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.fail;
import java.util.function.Consumer; import java.util.function.Consumer;
import java.util.function.Supplier; import java.util.function.Supplier;
import lombok.Builder;
import lombok.EqualsAndHashCode;
import lombok.RequiredArgsConstructor;
import lombok.ToString;
import lombok.experimental.Accessors;
import org.apache.commons.beanutils.BeanUtilsBean; import org.apache.commons.beanutils.BeanUtilsBean;
import org.apache.commons.beanutils.ConvertUtilsBean; import org.apache.commons.beanutils.ConvertUtilsBean;
import org.junit.Before; import org.junit.Before;
@ -32,11 +30,12 @@ import org.junit.Rule;
import org.junit.Test; import org.junit.Test;
import org.junit.rules.ExpectedException; import org.junit.rules.ExpectedException;
import lombok.Builder; import static org.hamcrest.CoreMatchers.containsString;
import lombok.EqualsAndHashCode; import static org.hamcrest.CoreMatchers.equalTo;
import lombok.RequiredArgsConstructor; import static org.hamcrest.CoreMatchers.instanceOf;
import lombok.ToString; import static org.hamcrest.CoreMatchers.nullValue;
import lombok.experimental.Accessors; import static org.junit.Assert.assertThat;
import static org.junit.Assert.fail;
public class BuilderDynaBeanTest { public class BuilderDynaBeanTest {
@ -109,8 +108,8 @@ public class BuilderDynaBeanTest {
@Test @Test
public void testComplexCreateAllParameters() throws Exception { public void testComplexCreateAllParameters() throws Exception {
TestComplexCreate expected = TestComplexCreate.create("real", TestComplexCreate expected = TestComplexCreate.create(
TestSimpleBuilder.builder().stringL1("l1").longVal(10L).build()); "real", TestSimpleBuilder.builder().stringL1("l1").longVal(10L).build());
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreate.class, convertUtilsBean); BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreate.class, convertUtilsBean);
utilsBean.setProperty(builderDynaBean, "[0]", expected.realName); utilsBean.setProperty(builderDynaBean, "[0]", expected.realName);
@ -136,8 +135,8 @@ public class BuilderDynaBeanTest {
@Test @Test
public void testComplexCreateComplexParameterOnly() throws Exception { public void testComplexCreateComplexParameterOnly() throws Exception {
TestComplexCreate expected = TestComplexCreate.create(null, TestComplexCreate expected = TestComplexCreate.create(
TestSimpleBuilder.builder().stringL1("l1").longVal(10L).build()); null, TestSimpleBuilder.builder().stringL1("l1").longVal(10L).build());
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreate.class, convertUtilsBean); BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreate.class, convertUtilsBean);
utilsBean.setProperty(builderDynaBean, "[1].stringL1", expected.test1.stringL1); utilsBean.setProperty(builderDynaBean, "[1].stringL1", expected.test1.stringL1);
@ -161,7 +160,8 @@ public class BuilderDynaBeanTest {
@Test @Test
public void testSimpleBuilderAllParameters() throws Exception { public void testSimpleBuilderAllParameters() throws Exception {
TestSimpleBuilder expected = TestSimpleBuilder.builder().stringL1("l1").longVal(10L).build(); TestSimpleBuilder expected =
TestSimpleBuilder.builder().stringL1("l1").longVal(10L).build();
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleBuilder.class, convertUtilsBean); BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleBuilder.class, convertUtilsBean);
utilsBean.setProperty(builderDynaBean, "stringL1", expected.stringL1); utilsBean.setProperty(builderDynaBean, "stringL1", expected.stringL1);
@ -213,12 +213,14 @@ public class BuilderDynaBeanTest {
@Test @Test
public void testComplexCreateSimpleBuilderVariantAllParameters() throws Exception { public void testComplexCreateSimpleBuilderVariantAllParameters() throws Exception {
TestSimpleBuilder variant = TestSimpleBuilder.builder().longVal(10L).stringL1("variant").build(); TestSimpleBuilder variant =
TestSimpleBuilder.builder().longVal(10L).stringL1("variant").build();
TestComplexCreateVariance expected = TestComplexCreateVariance.create("simple-builder", variant); TestComplexCreateVariance expected = TestComplexCreateVariance.create("simple-builder", variant);
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean); BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean);
utilsBean.setProperty(builderDynaBean, "[0]", expected.varianceName); utilsBean.setProperty(builderDynaBean, "[0]", expected.varianceName);
utilsBean.setProperty(builderDynaBean, "[1].class", expected.variant.getClass().getName()); utilsBean.setProperty(
builderDynaBean, "[1].class", expected.variant.getClass().getName());
utilsBean.setProperty(builderDynaBean, "[1].longVal", variant.longVal); utilsBean.setProperty(builderDynaBean, "[1].longVal", variant.longVal);
utilsBean.setProperty(builderDynaBean, "[1].stringL1", variant.stringL1); utilsBean.setProperty(builderDynaBean, "[1].stringL1", variant.stringL1);
@ -229,8 +231,11 @@ public class BuilderDynaBeanTest {
@Test @Test
public void testComplexCreateVariantBuilderAllParameters() throws Exception { public void testComplexCreateVariantBuilderAllParameters() throws Exception {
TestVariantBuilder variant = TestVariantBuilder.builder().variantBuilderName("variant-build").intClass(20) TestVariantBuilder variant = TestVariantBuilder.builder()
.testEnum(TestEnum.Blue).build(); .variantBuilderName("variant-build")
.intClass(20)
.testEnum(TestEnum.Blue)
.build();
TestComplexCreateVariance expected = TestComplexCreateVariance.create("builder-variant", variant); TestComplexCreateVariance expected = TestComplexCreateVariance.create("builder-variant", variant);
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean); BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean);
@ -264,13 +269,16 @@ public class BuilderDynaBeanTest {
@Test @Test
public void testComplexCreateVariantBuilderAllParametersPrefixWithJoiner() throws Exception { public void testComplexCreateVariantBuilderAllParametersPrefixWithJoiner() throws Exception {
TestVariantBuilder variant = TestVariantBuilder.builder().variantBuilderName("variant-build").intClass(20) TestVariantBuilder variant = TestVariantBuilder.builder()
.testEnum(TestEnum.Blue).build(); .variantBuilderName("variant-build")
.intClass(20)
.testEnum(TestEnum.Blue)
.build();
TestComplexCreateVariance expected = TestComplexCreateVariance.create("builder-variant-prefix", variant); TestComplexCreateVariance expected = TestComplexCreateVariance.create("builder-variant-prefix", variant);
String prefix = variant.getClass().getEnclosingClass().getName() + "$"; String prefix = variant.getClass().getEnclosingClass().getName() + "$";
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean, BuilderDynaBean builderDynaBean =
prefix); new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean, prefix);
utilsBean.setProperty(builderDynaBean, "[0]", expected.varianceName); utilsBean.setProperty(builderDynaBean, "[0]", expected.varianceName);
utilsBean.setProperty(builderDynaBean, "[1].class", variant.getClass().getSimpleName()); utilsBean.setProperty(builderDynaBean, "[1].class", variant.getClass().getSimpleName());
utilsBean.setProperty(builderDynaBean, "[1].variantBuilderName", variant.variantBuilderName); utilsBean.setProperty(builderDynaBean, "[1].variantBuilderName", variant.variantBuilderName);
@ -284,13 +292,16 @@ public class BuilderDynaBeanTest {
@Test @Test
public void testComplexCreateVariantBuilderAllParametersPrefixWithOutJoiner() throws Exception { public void testComplexCreateVariantBuilderAllParametersPrefixWithOutJoiner() throws Exception {
TestVariantBuilder variant = TestVariantBuilder.builder().variantBuilderName("variant-build").intClass(20) TestVariantBuilder variant = TestVariantBuilder.builder()
.testEnum(TestEnum.Blue).build(); .variantBuilderName("variant-build")
.intClass(20)
.testEnum(TestEnum.Blue)
.build();
TestComplexCreateVariance expected = TestComplexCreateVariance.create("builder-variant-prefix", variant); TestComplexCreateVariance expected = TestComplexCreateVariance.create("builder-variant-prefix", variant);
String prefix = variant.getClass().getEnclosingClass().getName(); String prefix = variant.getClass().getEnclosingClass().getName();
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean, BuilderDynaBean builderDynaBean =
prefix); new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean, prefix);
utilsBean.setProperty(builderDynaBean, "[0]", expected.varianceName); utilsBean.setProperty(builderDynaBean, "[0]", expected.varianceName);
utilsBean.setProperty(builderDynaBean, "[1].class", variant.getClass().getSimpleName()); utilsBean.setProperty(builderDynaBean, "[1].class", variant.getClass().getSimpleName());
utilsBean.setProperty(builderDynaBean, "[1].variantBuilderName", variant.variantBuilderName); utilsBean.setProperty(builderDynaBean, "[1].variantBuilderName", variant.variantBuilderName);
@ -330,11 +341,21 @@ public class BuilderDynaBeanTest {
@Test @Test
public void testComplexRootAllParameters() throws Exception { public void testComplexRootAllParameters() throws Exception {
TestSimpleBuilder simpleBuilder = TestSimpleBuilder.builder().stringL1("simple-l1").longVal(20L).build(); TestSimpleBuilder simpleBuilder =
TestRootClass expected = TestRootClass.builder().intVal(10).stringVal("root").testEnum(TestEnum.Red) TestSimpleBuilder.builder().stringL1("simple-l1").longVal(20L).build();
.testComplexCreate(TestComplexCreate.create("real", TestRootClass expected = TestRootClass.builder()
TestSimpleBuilder.builder().stringL1("complex-l1").longVal(10L).build())) .intVal(10)
.testSimpleBuilder(simpleBuilder).testSimpleCreate(TestSimpleCreate.create("first", "last")).build(); .stringVal("root")
.testEnum(TestEnum.Red)
.testComplexCreate(TestComplexCreate.create(
"real",
TestSimpleBuilder.builder()
.stringL1("complex-l1")
.longVal(10L)
.build()))
.testSimpleBuilder(simpleBuilder)
.testSimpleCreate(TestSimpleCreate.create("first", "last"))
.build();
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestRootClass.class, convertUtilsBean); BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestRootClass.class, convertUtilsBean);
@ -342,10 +363,10 @@ public class BuilderDynaBeanTest {
utilsBean.setProperty(builderDynaBean, "stringVal", expected.stringVal); utilsBean.setProperty(builderDynaBean, "stringVal", expected.stringVal);
utilsBean.setProperty(builderDynaBean, "testEnum", expected.testEnum); utilsBean.setProperty(builderDynaBean, "testEnum", expected.testEnum);
utilsBean.setProperty(builderDynaBean, "testComplexCreate.[0]", expected.testComplexCreate.realName); utilsBean.setProperty(builderDynaBean, "testComplexCreate.[0]", expected.testComplexCreate.realName);
utilsBean.setProperty(builderDynaBean, "testComplexCreate.[1].stringL1", utilsBean.setProperty(
expected.testComplexCreate.test1.stringL1); builderDynaBean, "testComplexCreate.[1].stringL1", expected.testComplexCreate.test1.stringL1);
utilsBean.setProperty(builderDynaBean, "testComplexCreate.[1].longVal", utilsBean.setProperty(
expected.testComplexCreate.test1.longVal); builderDynaBean, "testComplexCreate.[1].longVal", expected.testComplexCreate.test1.longVal);
utilsBean.setProperty(builderDynaBean, "testSimpleBuilder.class", TestSimpleBuilder.class.getName()); utilsBean.setProperty(builderDynaBean, "testSimpleBuilder.class", TestSimpleBuilder.class.getName());
utilsBean.setProperty(builderDynaBean, "testSimpleBuilder.stringL1", simpleBuilder.stringL1); utilsBean.setProperty(builderDynaBean, "testSimpleBuilder.stringL1", simpleBuilder.stringL1);
utilsBean.setProperty(builderDynaBean, "testSimpleBuilder.longVal", simpleBuilder.longVal); utilsBean.setProperty(builderDynaBean, "testSimpleBuilder.longVal", simpleBuilder.longVal);
@ -370,7 +391,11 @@ public class BuilderDynaBeanTest {
@Test @Test
public void testComplexRootTopLevelOnly() throws Exception { public void testComplexRootTopLevelOnly() throws Exception {
TestRootClass expected = TestRootClass.builder().intVal(10).stringVal("root").testEnum(TestEnum.Red).build(); TestRootClass expected = TestRootClass.builder()
.intVal(10)
.stringVal("root")
.testEnum(TestEnum.Red)
.build();
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestRootClass.class, convertUtilsBean); BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestRootClass.class, convertUtilsBean);
@ -385,12 +410,17 @@ public class BuilderDynaBeanTest {
@Test @Test
public void testSupplierNotUsed() throws Exception { public void testSupplierNotUsed() throws Exception {
TestVariantBuilder variant = TestVariantBuilder.builder().testEnum(TestEnum.Green).intClass(10) TestVariantBuilder variant = TestVariantBuilder.builder()
.variantBuilderName("variant-supplier").build(); .testEnum(TestEnum.Green)
TestSupplierClass expected = TestSupplierClass.builder().variantClass(variant).build(); .intClass(10)
.variantBuilderName("variant-supplier")
.build();
TestSupplierClass expected =
TestSupplierClass.builder().variantClass(variant).build();
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSupplierClass.class, convertUtilsBean); BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSupplierClass.class, convertUtilsBean);
utilsBean.setProperty(builderDynaBean, "variantClass.class", variant.getClass().getName()); utilsBean.setProperty(
builderDynaBean, "variantClass.class", variant.getClass().getName());
utilsBean.setProperty(builderDynaBean, "variantClass.testEnum", variant.testEnum); utilsBean.setProperty(builderDynaBean, "variantClass.testEnum", variant.testEnum);
utilsBean.setProperty(builderDynaBean, "variantClass.intClass", variant.intClass); utilsBean.setProperty(builderDynaBean, "variantClass.intClass", variant.intClass);
utilsBean.setProperty(builderDynaBean, "variantClass.variantBuilderName", variant.variantBuilderName); utilsBean.setProperty(builderDynaBean, "variantClass.variantBuilderName", variant.variantBuilderName);
@ -422,8 +452,11 @@ public class BuilderDynaBeanTest {
@Test @Test
public void testVariantBuildsToSuperType() throws Exception { public void testVariantBuildsToSuperType() throws Exception {
TestVariantBuilder expected = TestVariantBuilder.builder().intClass(10).testEnum(TestEnum.Green) TestVariantBuilder expected = TestVariantBuilder.builder()
.variantBuilderName("variant-super").build(); .intClass(10)
.testEnum(TestEnum.Green)
.variantBuilderName("variant-super")
.build();
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestInterface.class, convertUtilsBean); BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestInterface.class, convertUtilsBean);
utilsBean.setProperty(builderDynaBean, "class", expected.getClass().getName()); utilsBean.setProperty(builderDynaBean, "class", expected.getClass().getName());
@ -439,9 +472,11 @@ public class BuilderDynaBeanTest {
@Test @Test
public void testEmptyPropertyHandler() throws Exception { public void testEmptyPropertyHandler() throws Exception {
String emptyPropertyValue = "test-property"; String emptyPropertyValue = "test-property";
TestVariantCreate expected = TestVariantCreate.create(emptyPropertyValue, (long) emptyPropertyValue.length(), TestVariantCreate expected = TestVariantCreate.create(
emptyPropertyValue + "-vary"); emptyPropertyValue, (long) emptyPropertyValue.length(), emptyPropertyValue + "-vary");
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestInterface.class, convertUtilsBean, BuilderDynaBean builderDynaBean = new BuilderDynaBean(
TestInterface.class,
convertUtilsBean,
s -> TestVariantCreate.create(s, (long) s.length(), s + "-vary")); s -> TestVariantCreate.create(s, (long) s.length(), s + "-vary"));
utilsBean.setProperty(builderDynaBean, "", emptyPropertyValue); utilsBean.setProperty(builderDynaBean, "", emptyPropertyValue);
@ -455,8 +490,8 @@ public class BuilderDynaBeanTest {
thrown.expect(IllegalStateException.class); thrown.expect(IllegalStateException.class);
thrown.expectMessage(containsString("When a property handler is resolved further properties may not be set.")); thrown.expectMessage(containsString("When a property handler is resolved further properties may not be set."));
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestInterface.class, convertUtilsBean, BuilderDynaBean builderDynaBean = new BuilderDynaBean(
s -> TestVariantCreate.create("test", 10, "test")); TestInterface.class, convertUtilsBean, s -> TestVariantCreate.create("test", 10, "test"));
utilsBean.setProperty(builderDynaBean, "", "test"); utilsBean.setProperty(builderDynaBean, "", "test");
utilsBean.setProperty(builderDynaBean, "[0]", "test"); utilsBean.setProperty(builderDynaBean, "[0]", "test");
} }
@ -468,8 +503,8 @@ public class BuilderDynaBeanTest {
thrown.expectMessage(containsString(TestInterface.class.getName())); thrown.expectMessage(containsString(TestInterface.class.getName()));
thrown.expectMessage(containsString("cannot be assigned to")); thrown.expectMessage(containsString("cannot be assigned to"));
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestInterface.class, convertUtilsBean, BuilderDynaBean builderDynaBean =
s -> TestEnum.Green); new BuilderDynaBean(TestInterface.class, convertUtilsBean, s -> TestEnum.Green);
utilsBean.setProperty(builderDynaBean, "", "test"); utilsBean.setProperty(builderDynaBean, "", "test");
@ -478,8 +513,11 @@ public class BuilderDynaBeanTest {
@Test @Test
public void testSimpleArrayValues() throws Exception { public void testSimpleArrayValues() throws Exception {
SimpleArrayClassVariant expected = SimpleArrayClassVariant.builder().ints(new Integer[] { 1, 2, 3 }) SimpleArrayClassVariant expected = SimpleArrayClassVariant.builder()
.variantName("simple-array").longs(new Long[] { 1L, 2L, 3L }).strings(new String[] { "a", "b", "c" }) .ints(new Integer[] {1, 2, 3})
.variantName("simple-array")
.longs(new Long[] {1L, 2L, 3L})
.strings(new String[] {"a", "b", "c"})
.build(); .build();
BuilderDynaBean builderDynaBean = new BuilderDynaBean(SimpleArrayClassVariant.class, convertUtilsBean); BuilderDynaBean builderDynaBean = new BuilderDynaBean(SimpleArrayClassVariant.class, convertUtilsBean);
@ -503,12 +541,20 @@ public class BuilderDynaBeanTest {
@Test @Test
public void testComplexArrayValuesBuilder() throws Exception { public void testComplexArrayValuesBuilder() throws Exception {
TestVariantBuilder variant1 = TestVariantBuilder.builder().variantBuilderName("variant-1") TestVariantBuilder variant1 = TestVariantBuilder.builder()
.testEnum(TestEnum.Green).intClass(10).build(); .variantBuilderName("variant-1")
TestVariantBuilder variant2 = TestVariantBuilder.builder().variantBuilderName("variant-2") .testEnum(TestEnum.Green)
.testEnum(TestEnum.Blue).intClass(20).build(); .intClass(10)
ComplexArrayClassVariant expected = ComplexArrayClassVariant.builder().variantName("complex-test") .build();
.tests(new TestInterface[] { variant1, variant2 }).build(); TestVariantBuilder variant2 = TestVariantBuilder.builder()
.variantBuilderName("variant-2")
.testEnum(TestEnum.Blue)
.intClass(20)
.build();
ComplexArrayClassVariant expected = ComplexArrayClassVariant.builder()
.variantName("complex-test")
.tests(new TestInterface[] {variant1, variant2})
.build();
BuilderDynaBean builderDynaBean = new BuilderDynaBean(ComplexArrayClassVariant.class, convertUtilsBean); BuilderDynaBean builderDynaBean = new BuilderDynaBean(ComplexArrayClassVariant.class, convertUtilsBean);
@ -533,18 +579,22 @@ public class BuilderDynaBeanTest {
TestVariantCreate variant1 = TestVariantCreate.create("variant-1", 10L, "vary-1"); TestVariantCreate variant1 = TestVariantCreate.create("variant-1", 10L, "vary-1");
TestVariantCreate variant2 = TestVariantCreate.create("variant-2", 20L, "vary-2"); TestVariantCreate variant2 = TestVariantCreate.create("variant-2", 20L, "vary-2");
ComplexArrayClassVariant expected = ComplexArrayClassVariant.builder().variantName("create-test") ComplexArrayClassVariant expected = ComplexArrayClassVariant.builder()
.tests(new TestInterface[] { variant1, variant2 }).build(); .variantName("create-test")
.tests(new TestInterface[] {variant1, variant2})
.build();
BuilderDynaBean builderDynaBean = new BuilderDynaBean(ComplexArrayClassVariant.class, convertUtilsBean); BuilderDynaBean builderDynaBean = new BuilderDynaBean(ComplexArrayClassVariant.class, convertUtilsBean);
utilsBean.setProperty(builderDynaBean, "variantName", expected.variantName); utilsBean.setProperty(builderDynaBean, "variantName", expected.variantName);
utilsBean.setProperty(builderDynaBean, "tests[0].class", variant1.getClass().getName()); utilsBean.setProperty(
builderDynaBean, "tests[0].class", variant1.getClass().getName());
utilsBean.setProperty(builderDynaBean, "tests[0].[0]", variant1.variantCreateName); utilsBean.setProperty(builderDynaBean, "tests[0].[0]", variant1.variantCreateName);
utilsBean.setProperty(builderDynaBean, "tests[0].[1]", variant1.longClass); utilsBean.setProperty(builderDynaBean, "tests[0].[1]", variant1.longClass);
utilsBean.setProperty(builderDynaBean, "tests[0].[2]", variant1.varyString); utilsBean.setProperty(builderDynaBean, "tests[0].[2]", variant1.varyString);
utilsBean.setProperty(builderDynaBean, "tests[1].class", variant2.getClass().getName()); utilsBean.setProperty(
builderDynaBean, "tests[1].class", variant2.getClass().getName());
utilsBean.setProperty(builderDynaBean, "tests[1].[0]", variant2.variantCreateName); utilsBean.setProperty(builderDynaBean, "tests[1].[0]", variant2.variantCreateName);
utilsBean.setProperty(builderDynaBean, "tests[1].[1]", variant2.longClass); utilsBean.setProperty(builderDynaBean, "tests[1].[1]", variant2.longClass);
utilsBean.setProperty(builderDynaBean, "tests[1].[2]", variant2.varyString); utilsBean.setProperty(builderDynaBean, "tests[1].[2]", variant2.varyString);
@ -552,7 +602,6 @@ public class BuilderDynaBeanTest {
ComplexArrayClassVariant actual = builderDynaBean.build(ComplexArrayClassVariant.class); ComplexArrayClassVariant actual = builderDynaBean.build(ComplexArrayClassVariant.class);
assertThat(actual, equalTo(expected)); assertThat(actual, equalTo(expected));
} }
@Test @Test
@ -562,13 +611,18 @@ public class BuilderDynaBeanTest {
if (i % 2 == 0) { if (i % 2 == 0) {
variants[i] = TestVariantCreate.create("create-variant-" + i, i + 5, "vary-" + i); variants[i] = TestVariantCreate.create("create-variant-" + i, i + 5, "vary-" + i);
} else { } else {
variants[i] = TestVariantBuilder.builder().testEnum(TestEnum.values()[i % TestEnum.values().length]) variants[i] = TestVariantBuilder.builder()
.intClass(i).variantBuilderName("builder-variant-" + i).build(); .testEnum(TestEnum.values()[i % TestEnum.values().length])
.intClass(i)
.variantBuilderName("builder-variant-" + i)
.build();
} }
} }
ComplexArrayClassVariant expected = ComplexArrayClassVariant.builder().variantName("large-complex") ComplexArrayClassVariant expected = ComplexArrayClassVariant.builder()
.tests(variants).build(); .variantName("large-complex")
.tests(variants)
.build();
BuilderDynaBean builderDynaBean = new BuilderDynaBean(ComplexArrayClassVariant.class, convertUtilsBean); BuilderDynaBean builderDynaBean = new BuilderDynaBean(ComplexArrayClassVariant.class, convertUtilsBean);
@ -578,13 +632,15 @@ public class BuilderDynaBeanTest {
TestInterface variant = variants[i]; TestInterface variant = variants[i];
if (variant instanceof TestVariantCreate) { if (variant instanceof TestVariantCreate) {
TestVariantCreate create = (TestVariantCreate) variant; TestVariantCreate create = (TestVariantCreate) variant;
utilsBean.setProperty(builderDynaBean, prefix + "class", create.getClass().getName()); utilsBean.setProperty(
builderDynaBean, prefix + "class", create.getClass().getName());
utilsBean.setProperty(builderDynaBean, prefix + "[0]", create.variantCreateName); utilsBean.setProperty(builderDynaBean, prefix + "[0]", create.variantCreateName);
utilsBean.setProperty(builderDynaBean, prefix + "[1]", create.longClass); utilsBean.setProperty(builderDynaBean, prefix + "[1]", create.longClass);
utilsBean.setProperty(builderDynaBean, prefix + "[2]", create.varyString); utilsBean.setProperty(builderDynaBean, prefix + "[2]", create.varyString);
} else if (variant instanceof TestVariantBuilder) { } else if (variant instanceof TestVariantBuilder) {
TestVariantBuilder builder = (TestVariantBuilder) variant; TestVariantBuilder builder = (TestVariantBuilder) variant;
utilsBean.setProperty(builderDynaBean, prefix + "class", builder.getClass().getName()); utilsBean.setProperty(
builderDynaBean, prefix + "class", builder.getClass().getName());
utilsBean.setProperty(builderDynaBean, prefix + "variantBuilderName", builder.variantBuilderName); utilsBean.setProperty(builderDynaBean, prefix + "variantBuilderName", builder.variantBuilderName);
utilsBean.setProperty(builderDynaBean, prefix + "intClass", builder.intClass); utilsBean.setProperty(builderDynaBean, prefix + "intClass", builder.intClass);
utilsBean.setProperty(builderDynaBean, prefix + "testEnum", builder.testEnum); utilsBean.setProperty(builderDynaBean, prefix + "testEnum", builder.testEnum);
@ -667,25 +723,27 @@ public class BuilderDynaBeanTest {
@Test @Test
public void testAdditionalMutators() throws Exception { public void testAdditionalMutators() throws Exception {
TestSimpleBuilder expected = TestSimpleBuilder.builder().stringL1("test").longVal(10L).build(); TestSimpleBuilder expected =
TestSimpleBuilder.builder().stringL1("test").longVal(10L).build();
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleBuilder.class, convertUtilsBean); BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleBuilder.class, convertUtilsBean);
utilsBean.setProperty(builderDynaBean, "stringL1", expected.stringL1); utilsBean.setProperty(builderDynaBean, "stringL1", expected.stringL1);
TestSimpleBuilder actual = builderDynaBean.build(TestSimpleBuilder.class, TestSimpleBuilder actual =
b -> ((TestSimpleBuilder.TestSimpleBuilderBuilder) b).longVal(expected.longVal)); builderDynaBean.build(TestSimpleBuilder.class, b -> ((TestSimpleBuilder.TestSimpleBuilderBuilder) b)
.longVal(expected.longVal));
assertThat(actual, equalTo(expected)); assertThat(actual, equalTo(expected));
} }
public enum TestEnum { public enum TestEnum {
Red, Green, Blue Red,
Green,
Blue
} }
public interface TestInterface { public interface TestInterface {}
}
@Accessors(fluent = true) @Accessors(fluent = true)
@ToString @ToString
@ -838,7 +896,5 @@ public class BuilderDynaBeanTest {
} }
public String name = "default"; public String name = "default";
} }
}
}

View file

@ -15,18 +15,17 @@
package software.amazon.kinesis.multilang.config; package software.amazon.kinesis.multilang.config;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.junit.Assert.assertThat;
import java.util.Optional; import java.util.Optional;
import org.junit.Test;
import lombok.Builder; import lombok.Builder;
import lombok.EqualsAndHashCode; import lombok.EqualsAndHashCode;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
import lombok.experimental.Accessors; import lombok.experimental.Accessors;
import org.junit.Test;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.junit.Assert.assertThat;
public class ConfigurationSettableUtilsTest { public class ConfigurationSettableUtilsTest {
@ -44,7 +43,10 @@ public class ConfigurationSettableUtilsTest {
public void testPrimitivesSet() { public void testPrimitivesSet() {
ConfigResult expected = ConfigResult.builder().rawInt(10).rawLong(15L).build(); ConfigResult expected = ConfigResult.builder().rawInt(10).rawLong(15L).build();
ConfigObject configObject = ConfigObject.builder().rawInt(expected.rawInt).rawLong(expected.rawLong).build(); ConfigObject configObject = ConfigObject.builder()
.rawInt(expected.rawInt)
.rawLong(expected.rawLong)
.build();
ConfigResult actual = resolve(configObject); ConfigResult actual = resolve(configObject);
assertThat(actual, equalTo(expected)); assertThat(actual, equalTo(expected));
@ -52,10 +54,14 @@ public class ConfigurationSettableUtilsTest {
@Test @Test
public void testHeapValuesSet() { public void testHeapValuesSet() {
ConfigResult expected = ConfigResult.builder().name("test").boxedInt(10).boxedLong(15L).build(); ConfigResult expected =
ConfigResult.builder().name("test").boxedInt(10).boxedLong(15L).build();
ConfigObject configObject = ConfigObject.builder().name(expected.name).boxedInt(expected.boxedInt.intValue()) ConfigObject configObject = ConfigObject.builder()
.boxedLong(expected.boxedLong.longValue()).build(); .name(expected.name)
.boxedInt(expected.boxedInt.intValue())
.boxedLong(expected.boxedLong.longValue())
.build();
ConfigResult actual = resolve(configObject); ConfigResult actual = resolve(configObject);
assertThat(actual, equalTo(expected)); assertThat(actual, equalTo(expected));
@ -63,27 +69,39 @@ public class ConfigurationSettableUtilsTest {
@Test @Test
public void testComplexValuesSet() { public void testComplexValuesSet() {
ComplexValue complexValue = ComplexValue.builder().name("complex").value(10).build(); ComplexValue complexValue =
ConfigResult expected = ConfigResult.builder().complexValue(complexValue).build(); ComplexValue.builder().name("complex").value(10).build();
ConfigResult expected =
ConfigResult.builder().complexValue(complexValue).build();
ConfigObject configObject = ConfigObject.builder() ConfigObject configObject = ConfigObject.builder()
.complexValue(ComplexValue.builder().name(complexValue.name).value(complexValue.value).build()).build(); .complexValue(ComplexValue.builder()
.name(complexValue.name)
.value(complexValue.value)
.build())
.build();
ConfigResult actual = resolve(configObject); ConfigResult actual = resolve(configObject);
assertThat(actual, equalTo(expected)); assertThat(actual, equalTo(expected));
} }
@Test @Test
public void testOptionalValuesSet() { public void testOptionalValuesSet() {
ComplexValue complexValue = ComplexValue.builder().name("optional-complex").value(20).build(); ComplexValue complexValue =
ConfigResult expected = ConfigResult.builder().optionalString(Optional.of("test")) ComplexValue.builder().name("optional-complex").value(20).build();
.optionalInteger(Optional.of(10)).optionalLong(Optional.of(15L)) ConfigResult expected = ConfigResult.builder()
.optionalComplexValue(Optional.of(complexValue)).build(); .optionalString(Optional.of("test"))
.optionalInteger(Optional.of(10))
.optionalLong(Optional.of(15L))
.optionalComplexValue(Optional.of(complexValue))
.build();
ConfigObject configObject = ConfigObject.builder().optionalString(expected.optionalString.get()) ConfigObject configObject = ConfigObject.builder()
.optionalInteger(expected.optionalInteger.get()).optionalLong(expected.optionalLong.get()) .optionalString(expected.optionalString.get())
.optionalComplexValue(expected.optionalComplexValue.get()).build(); .optionalInteger(expected.optionalInteger.get())
.optionalLong(expected.optionalLong.get())
.optionalComplexValue(expected.optionalComplexValue.get())
.build();
ConfigResult actual = resolve(configObject); ConfigResult actual = resolve(configObject);
assertThat(actual, equalTo(expected)); assertThat(actual, equalTo(expected));
@ -91,20 +109,29 @@ public class ConfigurationSettableUtilsTest {
@Test @Test
public void testRenamedRawValues() { public void testRenamedRawValues() {
ComplexValue complexValue = ComplexValue.builder().name("renamed-complex").value(20).build(); ComplexValue complexValue =
ConfigResult expected = ConfigResult.builder().renamedString("renamed").renamedInt(10) ComplexValue.builder().name("renamed-complex").value(20).build();
.renamedOptionalString(Optional.of("renamed-optional")).renamedComplexValue(complexValue).build(); ConfigResult expected = ConfigResult.builder()
.renamedString("renamed")
.renamedInt(10)
.renamedOptionalString(Optional.of("renamed-optional"))
.renamedComplexValue(complexValue)
.build();
ConfigObject configObject = ConfigObject.builder().toRenameString(expected.renamedString) ConfigObject configObject = ConfigObject.builder()
.toRenameInt(expected.renamedInt).toRenameComplexValue(complexValue) .toRenameString(expected.renamedString)
.optionalToRename(expected.renamedOptionalString.get()).build(); .toRenameInt(expected.renamedInt)
.toRenameComplexValue(complexValue)
.optionalToRename(expected.renamedOptionalString.get())
.build();
ConfigResult actual = resolve(configObject); ConfigResult actual = resolve(configObject);
assertThat(actual, equalTo(expected)); assertThat(actual, equalTo(expected));
} }
private ConfigResult resolve(ConfigObject configObject) { private ConfigResult resolve(ConfigObject configObject) {
return ConfigurationSettableUtils.resolveFields(configObject, ConfigResult.builder().build()); return ConfigurationSettableUtils.resolveFields(
configObject, ConfigResult.builder().build());
} }
@Accessors(fluent = true) @Accessors(fluent = true)
@ -129,7 +156,6 @@ public class ConfigurationSettableUtilsTest {
private int renamedInt; private int renamedInt;
private Optional<String> renamedOptionalString; private Optional<String> renamedOptionalString;
private ComplexValue renamedComplexValue; private ComplexValue renamedComplexValue;
} }
@Accessors(fluent = true) @Accessors(fluent = true)
@ -145,35 +171,47 @@ public class ConfigurationSettableUtilsTest {
@ConfigurationSettable(configurationClass = ConfigResult.class) @ConfigurationSettable(configurationClass = ConfigResult.class)
private String name; private String name;
@ConfigurationSettable(configurationClass = ConfigResult.class) @ConfigurationSettable(configurationClass = ConfigResult.class)
private int rawInt; private int rawInt;
@ConfigurationSettable(configurationClass = ConfigResult.class) @ConfigurationSettable(configurationClass = ConfigResult.class)
private Integer boxedInt; private Integer boxedInt;
@ConfigurationSettable(configurationClass = ConfigResult.class) @ConfigurationSettable(configurationClass = ConfigResult.class)
private long rawLong; private long rawLong;
@ConfigurationSettable(configurationClass = ConfigResult.class) @ConfigurationSettable(configurationClass = ConfigResult.class)
private Long boxedLong; private Long boxedLong;
@ConfigurationSettable(configurationClass = ConfigResult.class) @ConfigurationSettable(configurationClass = ConfigResult.class)
private ComplexValue complexValue; private ComplexValue complexValue;
@ConfigurationSettable(configurationClass = ConfigResult.class, convertToOptional = true) @ConfigurationSettable(configurationClass = ConfigResult.class, convertToOptional = true)
private String optionalString; private String optionalString;
@ConfigurationSettable(configurationClass = ConfigResult.class, convertToOptional = true) @ConfigurationSettable(configurationClass = ConfigResult.class, convertToOptional = true)
private Integer optionalInteger; private Integer optionalInteger;
@ConfigurationSettable(configurationClass = ConfigResult.class, convertToOptional = true) @ConfigurationSettable(configurationClass = ConfigResult.class, convertToOptional = true)
private Long optionalLong; private Long optionalLong;
@ConfigurationSettable(configurationClass = ConfigResult.class, convertToOptional = true) @ConfigurationSettable(configurationClass = ConfigResult.class, convertToOptional = true)
private ComplexValue optionalComplexValue; private ComplexValue optionalComplexValue;
@ConfigurationSettable(configurationClass = ConfigResult.class, methodName = "renamedString") @ConfigurationSettable(configurationClass = ConfigResult.class, methodName = "renamedString")
private String toRenameString; private String toRenameString;
@ConfigurationSettable(configurationClass = ConfigResult.class, methodName = "renamedInt") @ConfigurationSettable(configurationClass = ConfigResult.class, methodName = "renamedInt")
private int toRenameInt; private int toRenameInt;
@ConfigurationSettable(configurationClass = ConfigResult.class, methodName = "renamedOptionalString", convertToOptional = true)
@ConfigurationSettable(
configurationClass = ConfigResult.class,
methodName = "renamedOptionalString",
convertToOptional = true)
private String optionalToRename; private String optionalToRename;
@ConfigurationSettable(configurationClass = ConfigResult.class, methodName = "renamedComplexValue") @ConfigurationSettable(configurationClass = ConfigResult.class, methodName = "renamedComplexValue")
private ComplexValue toRenameComplexValue; private ComplexValue toRenameComplexValue;
} }
}
}

View file

@ -14,12 +14,12 @@
*/ */
package software.amazon.kinesis.multilang.config; package software.amazon.kinesis.multilang.config;
import static org.junit.Assert.assertEquals;
import java.util.Date; import java.util.Date;
import org.junit.Test; import org.junit.Test;
import static org.junit.Assert.assertEquals;
public class DatePropertyValueDecoderTest { public class DatePropertyValueDecoderTest {
private DatePropertyValueDecoder decoder = new DatePropertyValueDecoder(); private DatePropertyValueDecoder decoder = new DatePropertyValueDecoder();

View file

@ -21,7 +21,6 @@ import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.mockito.Mock; import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner; import org.mockito.runners.MockitoJUnitRunner;
import software.amazon.awssdk.services.kinesis.KinesisAsyncClient; import software.amazon.awssdk.services.kinesis.KinesisAsyncClient;
import software.amazon.kinesis.retrieval.fanout.FanOutConfig; import software.amazon.kinesis.retrieval.fanout.FanOutConfig;
@ -50,18 +49,22 @@ public class FanoutConfigBeanTest {
MultiLangDaemonConfiguration configuration = new MultiLangDaemonConfiguration(utilsBean, convertUtilsBean); MultiLangDaemonConfiguration configuration = new MultiLangDaemonConfiguration(utilsBean, convertUtilsBean);
configuration.setStreamName("test-stream"); configuration.setStreamName("test-stream");
configuration.setApplicationName("test-application"); configuration.setApplicationName("test-application");
FanOutConfig fanOutConfig =fanoutConfigBean.build(kinesisAsyncClient, configuration); FanOutConfig fanOutConfig = fanoutConfigBean.build(kinesisAsyncClient, configuration);
assertThat(fanOutConfig.kinesisClient(), equalTo(kinesisAsyncClient)); assertThat(fanOutConfig.kinesisClient(), equalTo(kinesisAsyncClient));
assertThat(fanOutConfig.streamName(), equalTo(configuration.getStreamName())); assertThat(fanOutConfig.streamName(), equalTo(configuration.getStreamName()));
assertThat(fanOutConfig.applicationName(), equalTo(configuration.getApplicationName())); assertThat(fanOutConfig.applicationName(), equalTo(configuration.getApplicationName()));
assertThat(fanOutConfig.consumerArn(), equalTo(fanoutConfigBean.getConsumerArn())); assertThat(fanOutConfig.consumerArn(), equalTo(fanoutConfigBean.getConsumerArn()));
assertThat(fanOutConfig.consumerName(), equalTo(fanoutConfigBean.getConsumerName())); assertThat(fanOutConfig.consumerName(), equalTo(fanoutConfigBean.getConsumerName()));
assertThat(fanOutConfig.maxDescribeStreamConsumerRetries(), equalTo(fanoutConfigBean.getMaxDescribeStreamConsumerRetries())); assertThat(
assertThat(fanOutConfig.maxDescribeStreamSummaryRetries(), equalTo(fanoutConfigBean.getMaxDescribeStreamSummaryRetries())); fanOutConfig.maxDescribeStreamConsumerRetries(),
assertThat(fanOutConfig.registerStreamConsumerRetries(), equalTo(fanoutConfigBean.getRegisterStreamConsumerRetries())); equalTo(fanoutConfigBean.getMaxDescribeStreamConsumerRetries()));
assertThat(
fanOutConfig.maxDescribeStreamSummaryRetries(),
equalTo(fanoutConfigBean.getMaxDescribeStreamSummaryRetries()));
assertThat(
fanOutConfig.registerStreamConsumerRetries(),
equalTo(fanoutConfigBean.getRegisterStreamConsumerRetries()));
assertThat(fanOutConfig.retryBackoffMillis(), equalTo(fanoutConfigBean.getRetryBackoffMillis())); assertThat(fanOutConfig.retryBackoffMillis(), equalTo(fanoutConfigBean.getRetryBackoffMillis()));
} }
}
}

View file

@ -14,15 +14,6 @@
*/ */
package software.amazon.kinesis.multilang.config; package software.amazon.kinesis.multilang.config;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.nullValue;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.InputStream; import java.io.InputStream;
import java.net.URI; import java.net.URI;
@ -34,18 +25,25 @@ import java.util.Set;
import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.auth.BasicAWSCredentials;
import com.google.common.collect.ImmutableSet;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.commons.lang3.exception.ExceptionUtils;
import org.junit.Test; import org.junit.Test;
import com.google.common.collect.ImmutableSet;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.mockito.runners.MockitoJUnitRunner; import org.mockito.runners.MockitoJUnitRunner;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.kinesis.common.InitialPositionInStream; import software.amazon.kinesis.common.InitialPositionInStream;
import software.amazon.kinesis.metrics.MetricsLevel; import software.amazon.kinesis.metrics.MetricsLevel;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.nullValue;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@RunWith(MockitoJUnitRunner.class) @RunWith(MockitoJUnitRunner.class)
public class KinesisClientLibConfiguratorTest { public class KinesisClientLibConfiguratorTest {
@ -58,8 +56,14 @@ public class KinesisClientLibConfiguratorTest {
@Test @Test
public void testWithBasicSetup() { public void testWithBasicSetup() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a", MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
"applicationName = b", "AWSCredentialsProvider = " + credentialName1, "workerId = 123" }, '\n')); new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = " + credentialName1,
"workerId = 123"
},
'\n'));
assertEquals(config.getApplicationName(), "b"); assertEquals(config.getApplicationName(), "b");
assertEquals(config.getStreamName(), "a"); assertEquals(config.getStreamName(), "a");
assertEquals(config.getWorkerIdentifier(), "123"); assertEquals(config.getWorkerIdentifier(), "123");
@ -69,9 +73,16 @@ public class KinesisClientLibConfiguratorTest {
@Test @Test
public void testWithLongVariables() { public void testWithLongVariables() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "applicationName = app", MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
"streamName = 123", "AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2, new String[] {
"workerId = 123", "failoverTimeMillis = 100", "shardSyncIntervalMillis = 500" }, '\n')); "applicationName = app",
"streamName = 123",
"AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"workerId = 123",
"failoverTimeMillis = 100",
"shardSyncIntervalMillis = 500"
},
'\n'));
assertEquals(config.getApplicationName(), "app"); assertEquals(config.getApplicationName(), "app");
assertEquals(config.getStreamName(), "123"); assertEquals(config.getStreamName(), "123");
@ -83,9 +94,14 @@ public class KinesisClientLibConfiguratorTest {
@Test @Test
public void testWithInitialPositionInStreamExtended() { public void testWithInitialPositionInStreamExtended() {
long epochTimeInSeconds = 1617406032; long epochTimeInSeconds = 1617406032;
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "applicationName = app", MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
"streamName = 123", "AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2, new String[] {
"initialPositionInStreamExtended = " + epochTimeInSeconds}, '\n')); "applicationName = app",
"streamName = 123",
"AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"initialPositionInStreamExtended = " + epochTimeInSeconds
},
'\n'));
assertEquals(config.getInitialPositionInStreamExtended().getTimestamp(), new Date(epochTimeInSeconds * 1000L)); assertEquals(config.getInitialPositionInStreamExtended().getTimestamp(), new Date(epochTimeInSeconds * 1000L));
assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.AT_TIMESTAMP); assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.AT_TIMESTAMP);
@ -96,9 +112,14 @@ public class KinesisClientLibConfiguratorTest {
// AT_TIMESTAMP cannot be used as initialPositionInStream. If a user wants to specify AT_TIMESTAMP, // AT_TIMESTAMP cannot be used as initialPositionInStream. If a user wants to specify AT_TIMESTAMP,
// they must specify the time with initialPositionInStreamExtended. // they must specify the time with initialPositionInStreamExtended.
try { try {
getConfiguration(StringUtils.join(new String[] { "applicationName = app", getConfiguration(StringUtils.join(
"streamName = 123", "AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2, new String[] {
"initialPositionInStream = AT_TIMESTAMP"}, '\n')); "applicationName = app",
"streamName = 123",
"AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"initialPositionInStream = AT_TIMESTAMP"
},
'\n'));
fail("Should have thrown when initialPositionInStream is set to AT_TIMESTAMP"); fail("Should have thrown when initialPositionInStream is set to AT_TIMESTAMP");
} catch (Exception e) { } catch (Exception e) {
Throwable rootCause = ExceptionUtils.getRootCause(e); Throwable rootCause = ExceptionUtils.getRootCause(e);
@ -111,9 +132,14 @@ public class KinesisClientLibConfiguratorTest {
// initialPositionInStreamExtended takes a long value indicating seconds since epoch. If a non-long // initialPositionInStreamExtended takes a long value indicating seconds since epoch. If a non-long
// value is provided, the constructor should throw an IllegalArgumentException exception. // value is provided, the constructor should throw an IllegalArgumentException exception.
try { try {
getConfiguration(StringUtils.join(new String[] { "applicationName = app", getConfiguration(StringUtils.join(
"streamName = 123", "AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2, new String[] {
"initialPositionInStreamExtended = null"}, '\n')); "applicationName = app",
"streamName = 123",
"AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"initialPositionInStreamExtended = null"
},
'\n'));
fail("Should have thrown when initialPositionInStreamExtended is set to null"); fail("Should have thrown when initialPositionInStreamExtended is set to null");
} catch (Exception e) { } catch (Exception e) {
Throwable rootCause = ExceptionUtils.getRootCause(e); Throwable rootCause = ExceptionUtils.getRootCause(e);
@ -124,8 +150,13 @@ public class KinesisClientLibConfiguratorTest {
@Test @Test
public void testWithUnsupportedClientConfigurationVariables() { public void testWithUnsupportedClientConfigurationVariables() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join( MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
new String[] { "AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2, "workerId = id", new String[] {
"kinesisClientConfig = {}", "streamName = stream", "applicationName = b" }, "AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"workerId = id",
"kinesisClientConfig = {}",
"streamName = stream",
"applicationName = b"
},
'\n')); '\n'));
assertEquals(config.getApplicationName(), "b"); assertEquals(config.getApplicationName(), "b");
@ -136,10 +167,18 @@ public class KinesisClientLibConfiguratorTest {
@Test @Test
public void testWithIntVariables() { public void testWithIntVariables() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = kinesis", MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
"AWSCredentialsProvider = " + credentialName2 + ", " + credentialName1, "workerId = w123", new String[] {
"maxRecords = 10", "metricsMaxQueueSize = 20", "applicationName = kinesis", "streamName = kinesis",
"retryGetRecordsInSeconds = 2", "maxGetRecordsThreadPool = 1" }, '\n')); "AWSCredentialsProvider = " + credentialName2 + ", " + credentialName1,
"workerId = w123",
"maxRecords = 10",
"metricsMaxQueueSize = 20",
"applicationName = kinesis",
"retryGetRecordsInSeconds = 2",
"maxGetRecordsThreadPool = 1"
},
'\n'));
assertEquals(config.getApplicationName(), "kinesis"); assertEquals(config.getApplicationName(), "kinesis");
assertEquals(config.getStreamName(), "kinesis"); assertEquals(config.getStreamName(), "kinesis");
@ -152,9 +191,15 @@ public class KinesisClientLibConfiguratorTest {
@Test @Test
public void testWithBooleanVariables() { public void testWithBooleanVariables() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a", MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
"applicationName = b", "AWSCredentialsProvider = ABCD, " + credentialName1, "workerId = 0", new String[] {
"cleanupLeasesUponShardCompletion = false", "validateSequenceNumberBeforeCheckpointing = true" }, "streamName = a",
"applicationName = b",
"AWSCredentialsProvider = ABCD, " + credentialName1,
"workerId = 0",
"cleanupLeasesUponShardCompletion = false",
"validateSequenceNumberBeforeCheckpointing = true"
},
'\n')); '\n'));
assertEquals(config.getApplicationName(), "b"); assertEquals(config.getApplicationName(), "b");
@ -166,9 +211,16 @@ public class KinesisClientLibConfiguratorTest {
@Test @Test
public void testWithStringVariables() { public void testWithStringVariables() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a", MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
"applicationName = b", "AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 1", new String[] {
"kinesisEndpoint = https://kinesis", "metricsLevel = SUMMARY" }, '\n')); "streamName = a",
"applicationName = b",
"AWSCredentialsProvider = ABCD," + credentialName1,
"workerId = 1",
"kinesisEndpoint = https://kinesis",
"metricsLevel = SUMMARY"
},
'\n'));
assertEquals(config.getWorkerIdentifier(), "1"); assertEquals(config.getWorkerIdentifier(), "1");
assertEquals(config.getKinesisClient().get("endpointOverride"), URI.create("https://kinesis")); assertEquals(config.getKinesisClient().get("endpointOverride"), URI.create("https://kinesis"));
@ -177,38 +229,66 @@ public class KinesisClientLibConfiguratorTest {
@Test @Test
public void testWithSetVariables() { public void testWithSetVariables() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a", MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
"applicationName = b", "AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 1", new String[] {
"metricsEnabledDimensions = ShardId, WorkerIdentifier" }, '\n')); "streamName = a",
"applicationName = b",
"AWSCredentialsProvider = ABCD," + credentialName1,
"workerId = 1",
"metricsEnabledDimensions = ShardId, WorkerIdentifier"
},
'\n'));
Set<String> expectedMetricsEnabledDimensions = ImmutableSet.<String> builder() Set<String> expectedMetricsEnabledDimensions = ImmutableSet.<String>builder()
.add("ShardId", "WorkerIdentifier").build(); .add("ShardId", "WorkerIdentifier")
assertThat(new HashSet<>(Arrays.asList(config.getMetricsEnabledDimensions())), equalTo(expectedMetricsEnabledDimensions)); .build();
assertThat(
new HashSet<>(Arrays.asList(config.getMetricsEnabledDimensions())),
equalTo(expectedMetricsEnabledDimensions));
} }
@Test @Test
public void testWithInitialPositionInStreamTrimHorizon() { public void testWithInitialPositionInStreamTrimHorizon() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a", MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
"applicationName = b", "AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 123", new String[] {
"initialPositionInStream = TriM_Horizon" }, '\n')); "streamName = a",
"applicationName = b",
"AWSCredentialsProvider = ABCD," + credentialName1,
"workerId = 123",
"initialPositionInStream = TriM_Horizon"
},
'\n'));
assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.TRIM_HORIZON); assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.TRIM_HORIZON);
} }
@Test @Test
public void testWithInitialPositionInStreamLatest() { public void testWithInitialPositionInStreamLatest() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a", MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
"applicationName = b", "AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 123", new String[] {
"initialPositionInStream = LateSt" }, '\n')); "streamName = a",
"applicationName = b",
"AWSCredentialsProvider = ABCD," + credentialName1,
"workerId = 123",
"initialPositionInStream = LateSt"
},
'\n'));
assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.LATEST); assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.LATEST);
} }
@Test @Test
public void testSkippingNonKCLVariables() { public void testSkippingNonKCLVariables() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a", MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
"applicationName = b", "AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 123", new String[] {
"initialPositionInStream = TriM_Horizon", "abc = 1" }, '\n')); "streamName = a",
"applicationName = b",
"AWSCredentialsProvider = ABCD," + credentialName1,
"workerId = 123",
"initialPositionInStream = TriM_Horizon",
"abc = 1"
},
'\n'));
assertEquals(config.getApplicationName(), "b"); assertEquals(config.getApplicationName(), "b");
assertEquals(config.getStreamName(), "a"); assertEquals(config.getStreamName(), "a");
@ -218,33 +298,61 @@ public class KinesisClientLibConfiguratorTest {
@Test @Test
public void testEmptyOptionalVariables() { public void testEmptyOptionalVariables() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a", MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
"applicationName = b", "AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 123", new String[] {
"initialPositionInStream = TriM_Horizon", "maxGetRecordsThreadPool = 1" }, '\n')); "streamName = a",
"applicationName = b",
"AWSCredentialsProvider = ABCD," + credentialName1,
"workerId = 123",
"initialPositionInStream = TriM_Horizon",
"maxGetRecordsThreadPool = 1"
},
'\n'));
assertThat(config.getMaxGetRecordsThreadPool(), equalTo(1)); assertThat(config.getMaxGetRecordsThreadPool(), equalTo(1));
assertThat(config.getRetryGetRecordsInSeconds(), nullValue()); assertThat(config.getRetryGetRecordsInSeconds(), nullValue());
} }
@Test @Test
public void testWithZeroValue() { public void testWithZeroValue() {
String test = StringUtils.join(new String[] { "streamName = a", "applicationName = b", String test = StringUtils.join(
"AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 123", new String[] {
"initialPositionInStream = TriM_Horizon", "maxGetRecordsThreadPool = 0", "streamName = a",
"retryGetRecordsInSeconds = 0" }, '\n'); "applicationName = b",
"AWSCredentialsProvider = ABCD," + credentialName1,
"workerId = 123",
"initialPositionInStream = TriM_Horizon",
"maxGetRecordsThreadPool = 0",
"retryGetRecordsInSeconds = 0"
},
'\n');
getConfiguration(test); getConfiguration(test);
} }
@Test @Test
public void testWithInvalidIntValue() { public void testWithInvalidIntValue() {
String test = StringUtils.join(new String[] { "streamName = a", "applicationName = b", String test = StringUtils.join(
"AWSCredentialsProvider = " + credentialName1, "workerId = 123", "failoverTimeMillis = 100nf" }, '\n'); new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = " + credentialName1,
"workerId = 123",
"failoverTimeMillis = 100nf"
},
'\n');
getConfiguration(test); getConfiguration(test);
} }
@Test @Test
public void testWithNegativeIntValue() { public void testWithNegativeIntValue() {
String test = StringUtils.join(new String[] { "streamName = a", "applicationName = b", String test = StringUtils.join(
"AWSCredentialsProvider = " + credentialName1, "workerId = 123", "failoverTimeMillis = -12" }, '\n'); new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = " + credentialName1,
"workerId = 123",
"failoverTimeMillis = -12"
},
'\n');
// separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement // separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement
getConfiguration(test); getConfiguration(test);
@ -252,8 +360,15 @@ public class KinesisClientLibConfiguratorTest {
@Test(expected = IllegalArgumentException.class) @Test(expected = IllegalArgumentException.class)
public void testWithMissingCredentialsProvider() { public void testWithMissingCredentialsProvider() {
String test = StringUtils.join(new String[] { "streamName = a", "applicationName = b", "workerId = 123", String test = StringUtils.join(
"failoverTimeMillis = 100", "shardSyncIntervalMillis = 500" }, '\n'); new String[] {
"streamName = a",
"applicationName = b",
"workerId = 123",
"failoverTimeMillis = 100",
"shardSyncIntervalMillis = 500"
},
'\n');
// separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement // separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement
getConfiguration(test); getConfiguration(test);
@ -262,8 +377,13 @@ public class KinesisClientLibConfiguratorTest {
@Test @Test
public void testWithMissingWorkerId() { public void testWithMissingWorkerId() {
String test = StringUtils.join( String test = StringUtils.join(
new String[] { "streamName = a", "applicationName = b", "AWSCredentialsProvider = " + credentialName1, new String[] {
"failoverTimeMillis = 100", "shardSyncIntervalMillis = 500" }, "streamName = a",
"applicationName = b",
"AWSCredentialsProvider = " + credentialName1,
"failoverTimeMillis = 100",
"shardSyncIntervalMillis = 500"
},
'\n'); '\n');
MultiLangDaemonConfiguration config = getConfiguration(test); MultiLangDaemonConfiguration config = getConfiguration(test);
@ -274,46 +394,63 @@ public class KinesisClientLibConfiguratorTest {
@Test(expected = NullPointerException.class) @Test(expected = NullPointerException.class)
public void testWithMissingStreamNameAndMissingStreamArn() { public void testWithMissingStreamNameAndMissingStreamArn() {
String test = StringUtils.join(new String[] { String test = StringUtils.join(
"applicationName = b", new String[] {
"AWSCredentialsProvider = " + credentialName1, "applicationName = b",
"workerId = 123", "AWSCredentialsProvider = " + credentialName1,
"failoverTimeMillis = 100" }, "workerId = 123",
"failoverTimeMillis = 100"
},
'\n'); '\n');
getConfiguration(test); getConfiguration(test);
} }
@Test(expected = IllegalArgumentException.class) @Test(expected = IllegalArgumentException.class)
public void testWithEmptyStreamNameAndMissingStreamArn() { public void testWithEmptyStreamNameAndMissingStreamArn() {
String test = StringUtils.join(new String[] { String test = StringUtils.join(
"applicationName = b", new String[] {
"AWSCredentialsProvider = " + credentialName1, "applicationName = b",
"workerId = 123", "AWSCredentialsProvider = " + credentialName1,
"failoverTimeMillis = 100", "workerId = 123",
"streamName = ", "failoverTimeMillis = 100",
"streamArn = "}, "streamName = ",
"streamArn = "
},
'\n'); '\n');
getConfiguration(test); getConfiguration(test);
} }
@Test(expected = NullPointerException.class) @Test(expected = NullPointerException.class)
public void testWithMissingApplicationName() { public void testWithMissingApplicationName() {
String test = StringUtils.join(new String[] { "streamName = a", "AWSCredentialsProvider = " + credentialName1, String test = StringUtils.join(
"workerId = 123", "failoverTimeMillis = 100" }, '\n'); new String[] {
"streamName = a",
"AWSCredentialsProvider = " + credentialName1,
"workerId = 123",
"failoverTimeMillis = 100"
},
'\n');
getConfiguration(test); getConfiguration(test);
} }
@Test @Test
public void testWithAWSCredentialsFailed() { public void testWithAWSCredentialsFailed() {
String test = StringUtils.join( String test = StringUtils.join(
new String[] { "streamName = a", "applicationName = b", "AWSCredentialsProvider = " + credentialName2, new String[] {
"failoverTimeMillis = 100", "shardSyncIntervalMillis = 500" }, "streamName = a",
"applicationName = b",
"AWSCredentialsProvider = " + credentialName2,
"failoverTimeMillis = 100",
"shardSyncIntervalMillis = 500"
},
'\n'); '\n');
MultiLangDaemonConfiguration config = getConfiguration(test); MultiLangDaemonConfiguration config = getConfiguration(test);
// separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement // separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement
try { try {
config.getKinesisCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials(); config.getKinesisCredentialsProvider()
.build(AwsCredentialsProvider.class)
.resolveCredentials();
fail("expect failure with wrong credentials provider"); fail("expect failure with wrong credentials provider");
} catch (Exception e) { } catch (Exception e) {
// succeed // succeed
@ -323,39 +460,63 @@ public class KinesisClientLibConfiguratorTest {
// TODO: fix this test // TODO: fix this test
@Test @Test
public void testWithDifferentAWSCredentialsForDynamoDBAndCloudWatch() { public void testWithDifferentAWSCredentialsForDynamoDBAndCloudWatch() {
String test = StringUtils.join(new String[] { "streamName = a", "applicationName = b", String test = StringUtils.join(
"AWSCredentialsProvider = " + credentialNameKinesis, new String[] {
"AWSCredentialsProviderDynamoDB = " + credentialNameDynamoDB, "streamName = a",
"AWSCredentialsProviderCloudWatch = " + credentialNameCloudWatch, "failoverTimeMillis = 100", "applicationName = b",
"shardSyncIntervalMillis = 500" }, '\n'); "AWSCredentialsProvider = " + credentialNameKinesis,
"AWSCredentialsProviderDynamoDB = " + credentialNameDynamoDB,
"AWSCredentialsProviderCloudWatch = " + credentialNameCloudWatch,
"failoverTimeMillis = 100",
"shardSyncIntervalMillis = 500"
},
'\n');
// separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement // separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement
final MultiLangDaemonConfiguration config = getConfiguration(test); final MultiLangDaemonConfiguration config = getConfiguration(test);
config.getKinesisCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials(); config.getKinesisCredentialsProvider()
config.getDynamoDBCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials(); .build(AwsCredentialsProvider.class)
config.getCloudWatchCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials(); .resolveCredentials();
config.getDynamoDBCredentialsProvider()
.build(AwsCredentialsProvider.class)
.resolveCredentials();
config.getCloudWatchCredentialsProvider()
.build(AwsCredentialsProvider.class)
.resolveCredentials();
} }
// TODO: fix this test // TODO: fix this test
@Test @Test
public void testWithDifferentAWSCredentialsForDynamoDBAndCloudWatchFailed() { public void testWithDifferentAWSCredentialsForDynamoDBAndCloudWatchFailed() {
String test = StringUtils.join(new String[] { "streamName = a", "applicationName = b", String test = StringUtils.join(
"AWSCredentialsProvider = " + credentialNameKinesis, new String[] {
"AWSCredentialsProviderDynamoDB = " + credentialName2, "streamName = a",
"AWSCredentialsProviderCloudWatch = " + credentialName2, "failoverTimeMillis = 100", "applicationName = b",
"shardSyncIntervalMillis = 500" }, '\n'); "AWSCredentialsProvider = " + credentialNameKinesis,
"AWSCredentialsProviderDynamoDB = " + credentialName2,
"AWSCredentialsProviderCloudWatch = " + credentialName2,
"failoverTimeMillis = 100",
"shardSyncIntervalMillis = 500"
},
'\n');
// separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement // separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement
final MultiLangDaemonConfiguration config = getConfiguration(test); final MultiLangDaemonConfiguration config = getConfiguration(test);
config.getKinesisCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials(); config.getKinesisCredentialsProvider()
.build(AwsCredentialsProvider.class)
.resolveCredentials();
try { try {
config.getDynamoDBCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials(); config.getDynamoDBCredentialsProvider()
.build(AwsCredentialsProvider.class)
.resolveCredentials();
fail("DynamoDB credential providers should fail."); fail("DynamoDB credential providers should fail.");
} catch (Exception e) { } catch (Exception e) {
// succeed // succeed
} }
try { try {
config.getCloudWatchCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials(); config.getCloudWatchCredentialsProvider()
.build(AwsCredentialsProvider.class)
.resolveCredentials();
fail("CloudWatch credential providers should fail."); fail("CloudWatch credential providers should fail.");
} catch (Exception e) { } catch (Exception e) {
// succeed // succeed
@ -373,9 +534,7 @@ public class KinesisClientLibConfiguratorTest {
} }
@Override @Override
public void refresh() { public void refresh() {}
}
} }
/** /**
@ -389,9 +548,7 @@ public class KinesisClientLibConfiguratorTest {
} }
@Override @Override
public void refresh() { public void refresh() {}
}
} }
/** /**
@ -405,9 +562,7 @@ public class KinesisClientLibConfiguratorTest {
} }
@Override @Override
public void refresh() { public void refresh() {}
}
} }
/** /**
@ -421,9 +576,7 @@ public class KinesisClientLibConfiguratorTest {
} }
@Override @Override
public void refresh() { public void refresh() {}
}
} }
/** /**
@ -437,13 +590,11 @@ public class KinesisClientLibConfiguratorTest {
} }
@Override @Override
public void refresh() { public void refresh() {}
}
} }
private MultiLangDaemonConfiguration getConfiguration(String configString) { private MultiLangDaemonConfiguration getConfiguration(String configString) {
InputStream input = new ByteArrayInputStream(configString.getBytes()); InputStream input = new ByteArrayInputStream(configString.getBytes());
return configurator.getConfiguration(input); return configurator.getConfiguration(input);
} }
} }

View file

@ -15,12 +15,6 @@
package software.amazon.kinesis.multilang.config; package software.amazon.kinesis.multilang.config;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import org.apache.commons.beanutils.BeanUtilsBean; import org.apache.commons.beanutils.BeanUtilsBean;
import org.apache.commons.beanutils.ConvertUtilsBean; import org.apache.commons.beanutils.ConvertUtilsBean;
import org.junit.After; import org.junit.After;
@ -31,12 +25,17 @@ import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.mockito.Mock; import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner; import org.mockito.runners.MockitoJUnitRunner;
import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
import software.amazon.kinesis.processor.ShardRecordProcessorFactory; import software.amazon.kinesis.processor.ShardRecordProcessorFactory;
import software.amazon.kinesis.retrieval.fanout.FanOutConfig; import software.amazon.kinesis.retrieval.fanout.FanOutConfig;
import software.amazon.kinesis.retrieval.polling.PollingConfig; import software.amazon.kinesis.retrieval.polling.PollingConfig;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
@RunWith(MockitoJUnitRunner.class) @RunWith(MockitoJUnitRunner.class)
public class MultiLangDaemonConfigurationTest { public class MultiLangDaemonConfigurationTest {
@ -69,7 +68,6 @@ public class MultiLangDaemonConfigurationTest {
} }
} }
public MultiLangDaemonConfiguration baseConfiguration() { public MultiLangDaemonConfiguration baseConfiguration() {
MultiLangDaemonConfiguration configuration = new MultiLangDaemonConfiguration(utilsBean, convertUtilsBean); MultiLangDaemonConfiguration configuration = new MultiLangDaemonConfiguration(utilsBean, convertUtilsBean);
configuration.setApplicationName("Test"); configuration.setApplicationName("Test");
@ -84,8 +82,8 @@ public class MultiLangDaemonConfigurationTest {
MultiLangDaemonConfiguration configuration = baseConfiguration(); MultiLangDaemonConfiguration configuration = baseConfiguration();
configuration.setMaxLeasesForWorker(10); configuration.setMaxLeasesForWorker(10);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
.resolvedConfiguration(shardRecordProcessorFactory); configuration.resolvedConfiguration(shardRecordProcessorFactory);
assertThat(resolvedConfiguration.leaseManagementConfig.maxLeasesForWorker(), equalTo(10)); assertThat(resolvedConfiguration.leaseManagementConfig.maxLeasesForWorker(), equalTo(10));
} }
@ -95,8 +93,8 @@ public class MultiLangDaemonConfigurationTest {
MultiLangDaemonConfiguration configuration = baseConfiguration(); MultiLangDaemonConfiguration configuration = baseConfiguration();
configuration.setEnablePriorityLeaseAssignment(false); configuration.setEnablePriorityLeaseAssignment(false);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration.resolvedConfiguration( MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
shardRecordProcessorFactory); configuration.resolvedConfiguration(shardRecordProcessorFactory);
assertThat(resolvedConfiguration.leaseManagementConfig.enablePriorityLeaseAssignment(), equalTo(false)); assertThat(resolvedConfiguration.leaseManagementConfig.enablePriorityLeaseAssignment(), equalTo(false));
} }
@ -105,11 +103,11 @@ public class MultiLangDaemonConfigurationTest {
public void testDefaultRetrievalConfig() { public void testDefaultRetrievalConfig() {
MultiLangDaemonConfiguration configuration = baseConfiguration(); MultiLangDaemonConfiguration configuration = baseConfiguration();
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
.resolvedConfiguration(shardRecordProcessorFactory); configuration.resolvedConfiguration(shardRecordProcessorFactory);
assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), assertThat(
instanceOf(FanOutConfig.class)); resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(FanOutConfig.class));
} }
@Test @Test
@ -118,18 +116,20 @@ public class MultiLangDaemonConfigurationTest {
configuration.setMaxRecords(10); configuration.setMaxRecords(10);
configuration.setIdleTimeBetweenReadsInMillis(60000); configuration.setIdleTimeBetweenReadsInMillis(60000);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
.resolvedConfiguration(shardRecordProcessorFactory); configuration.resolvedConfiguration(shardRecordProcessorFactory);
assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), assertThat(
instanceOf(PollingConfig.class)); resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(PollingConfig.class));
assertEquals(10, assertEquals(
((PollingConfig) resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig()).maxRecords()); 10,
assertEquals(60000, ((PollingConfig) resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig()).maxRecords());
((PollingConfig) resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig()) assertEquals(
.idleTimeBetweenReadsInMillis()); 60000,
((PollingConfig) resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig())
.idleTimeBetweenReadsInMillis());
assertTrue(((PollingConfig) resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig()) assertTrue(((PollingConfig) resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig())
.usePollingConfigIdleTimeValue()); .usePollingConfigIdleTimeValue());
} }
@Test @Test
@ -137,11 +137,11 @@ public class MultiLangDaemonConfigurationTest {
MultiLangDaemonConfiguration configuration = baseConfiguration(); MultiLangDaemonConfiguration configuration = baseConfiguration();
configuration.setRetrievalMode(RetrievalMode.FANOUT); configuration.setRetrievalMode(RetrievalMode.FANOUT);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
.resolvedConfiguration(shardRecordProcessorFactory); configuration.resolvedConfiguration(shardRecordProcessorFactory);
assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), assertThat(
instanceOf(FanOutConfig.class)); resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(FanOutConfig.class));
} }
@Test @Test
@ -149,37 +149,39 @@ public class MultiLangDaemonConfigurationTest {
MultiLangDaemonConfiguration configuration = baseConfiguration(); MultiLangDaemonConfiguration configuration = baseConfiguration();
configuration.setRetrievalMode(RetrievalMode.POLLING); configuration.setRetrievalMode(RetrievalMode.POLLING);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
.resolvedConfiguration(shardRecordProcessorFactory); configuration.resolvedConfiguration(shardRecordProcessorFactory);
assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), assertThat(
instanceOf(PollingConfig.class)); resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(PollingConfig.class));
} }
@Test @Test
public void testRetrievalModeSetForPollingString() throws Exception { public void testRetrievalModeSetForPollingString() throws Exception {
MultiLangDaemonConfiguration configuration = baseConfiguration(); MultiLangDaemonConfiguration configuration = baseConfiguration();
utilsBean.setProperty(configuration, "retrievalMode", RetrievalMode.POLLING.name().toLowerCase()); utilsBean.setProperty(
configuration, "retrievalMode", RetrievalMode.POLLING.name().toLowerCase());
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
.resolvedConfiguration(shardRecordProcessorFactory); configuration.resolvedConfiguration(shardRecordProcessorFactory);
assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), assertThat(
instanceOf(PollingConfig.class)); resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(PollingConfig.class));
} }
@Test @Test
public void testRetrievalModeSetForFanoutString() throws Exception { public void testRetrievalModeSetForFanoutString() throws Exception {
MultiLangDaemonConfiguration configuration = baseConfiguration(); MultiLangDaemonConfiguration configuration = baseConfiguration();
utilsBean.setProperty(configuration, "retrievalMode", RetrievalMode.FANOUT.name().toLowerCase()); utilsBean.setProperty(
configuration, "retrievalMode", RetrievalMode.FANOUT.name().toLowerCase());
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
.resolvedConfiguration(shardRecordProcessorFactory); configuration.resolvedConfiguration(shardRecordProcessorFactory);
assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), assertThat(
instanceOf(FanOutConfig.class)); resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(FanOutConfig.class));
} }
@Test @Test
@ -196,7 +198,7 @@ public class MultiLangDaemonConfigurationTest {
// TODO : Enable this test once https://github.com/awslabs/amazon-kinesis-client/issues/692 is resolved // TODO : Enable this test once https://github.com/awslabs/amazon-kinesis-client/issues/692 is resolved
public void testmetricsEnabledDimensions() { public void testmetricsEnabledDimensions() {
MultiLangDaemonConfiguration configuration = baseConfiguration(); MultiLangDaemonConfiguration configuration = baseConfiguration();
configuration.setMetricsEnabledDimensions(new String[]{"Operation"}); configuration.setMetricsEnabledDimensions(new String[] {"Operation"});
configuration.resolvedConfiguration(shardRecordProcessorFactory); configuration.resolvedConfiguration(shardRecordProcessorFactory);
} }
@ -209,14 +211,14 @@ public class MultiLangDaemonConfigurationTest {
configuration.setRetrievalMode(RetrievalMode.FANOUT); configuration.setRetrievalMode(RetrievalMode.FANOUT);
configuration.getFanoutConfig().setConsumerArn(consumerArn); configuration.getFanoutConfig().setConsumerArn(consumerArn);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
.resolvedConfiguration(shardRecordProcessorFactory); configuration.resolvedConfiguration(shardRecordProcessorFactory);
assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), assertThat(
instanceOf(FanOutConfig.class)); resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(FanOutConfig.class));
FanOutConfig fanOutConfig = (FanOutConfig) resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(); FanOutConfig fanOutConfig =
(FanOutConfig) resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig();
assertThat(fanOutConfig.consumerArn(), equalTo(consumerArn)); assertThat(fanOutConfig.consumerArn(), equalTo(consumerArn));
} }
}
}

View file

@ -15,6 +15,8 @@
package software.amazon.kinesis.multilang.config; package software.amazon.kinesis.multilang.config;
import java.util.Optional;
import org.apache.commons.beanutils.BeanUtilsBean; import org.apache.commons.beanutils.BeanUtilsBean;
import org.apache.commons.beanutils.ConvertUtilsBean; import org.apache.commons.beanutils.ConvertUtilsBean;
import org.junit.Test; import org.junit.Test;
@ -24,8 +26,6 @@ import org.mockito.runners.MockitoJUnitRunner;
import software.amazon.awssdk.services.kinesis.KinesisAsyncClient; import software.amazon.awssdk.services.kinesis.KinesisAsyncClient;
import software.amazon.kinesis.retrieval.polling.PollingConfig; import software.amazon.kinesis.retrieval.polling.PollingConfig;
import java.util.Optional;
import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.equalTo;
import static org.junit.Assert.assertThat; import static org.junit.Assert.assertThat;
@ -46,17 +46,23 @@ public class PollingConfigBeanTest {
ConvertUtilsBean convertUtilsBean = new ConvertUtilsBean(); ConvertUtilsBean convertUtilsBean = new ConvertUtilsBean();
BeanUtilsBean utilsBean = new BeanUtilsBean(convertUtilsBean); BeanUtilsBean utilsBean = new BeanUtilsBean(convertUtilsBean);
MultiLangDaemonConfiguration multiLangDaemonConfiguration = new MultiLangDaemonConfiguration(utilsBean, convertUtilsBean); MultiLangDaemonConfiguration multiLangDaemonConfiguration =
new MultiLangDaemonConfiguration(utilsBean, convertUtilsBean);
multiLangDaemonConfiguration.setStreamName("test-stream"); multiLangDaemonConfiguration.setStreamName("test-stream");
PollingConfig pollingConfig = pollingConfigBean.build(kinesisAsyncClient, multiLangDaemonConfiguration); PollingConfig pollingConfig = pollingConfigBean.build(kinesisAsyncClient, multiLangDaemonConfiguration);
assertThat(pollingConfig.kinesisClient(), equalTo(kinesisAsyncClient)); assertThat(pollingConfig.kinesisClient(), equalTo(kinesisAsyncClient));
assertThat(pollingConfig.streamName(), equalTo(multiLangDaemonConfiguration.getStreamName())); assertThat(pollingConfig.streamName(), equalTo(multiLangDaemonConfiguration.getStreamName()));
assertThat(pollingConfig.idleTimeBetweenReadsInMillis(), equalTo(pollingConfigBean.getIdleTimeBetweenReadsInMillis())); assertThat(
assertThat(pollingConfig.maxGetRecordsThreadPool(), equalTo(Optional.of(pollingConfigBean.getMaxGetRecordsThreadPool()))); pollingConfig.idleTimeBetweenReadsInMillis(),
equalTo(pollingConfigBean.getIdleTimeBetweenReadsInMillis()));
assertThat(
pollingConfig.maxGetRecordsThreadPool(),
equalTo(Optional.of(pollingConfigBean.getMaxGetRecordsThreadPool())));
assertThat(pollingConfig.maxRecords(), equalTo(pollingConfigBean.getMaxRecords())); assertThat(pollingConfig.maxRecords(), equalTo(pollingConfigBean.getMaxRecords()));
assertThat(pollingConfig.retryGetRecordsInSeconds(), equalTo(Optional.of(pollingConfigBean.getRetryGetRecordsInSeconds()))); assertThat(
pollingConfig.retryGetRecordsInSeconds(),
equalTo(Optional.of(pollingConfigBean.getRetryGetRecordsInSeconds())));
} }
}
}

View file

@ -15,11 +15,6 @@
package software.amazon.kinesis.multilang.messages; package software.amazon.kinesis.multilang.messages;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.nullValue;
import static org.hamcrest.CoreMatchers.sameInstance;
import static org.junit.Assert.assertThat;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.time.Instant; import java.time.Instant;
import java.util.Arrays; import java.util.Arrays;
@ -31,9 +26,13 @@ import org.hamcrest.Description;
import org.hamcrest.Matcher; import org.hamcrest.Matcher;
import org.hamcrest.TypeSafeDiagnosingMatcher; import org.hamcrest.TypeSafeDiagnosingMatcher;
import org.junit.Test; import org.junit.Test;
import software.amazon.kinesis.retrieval.KinesisClientRecord; import software.amazon.kinesis.retrieval.KinesisClientRecord;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.nullValue;
import static org.hamcrest.CoreMatchers.sameInstance;
import static org.junit.Assert.assertThat;
public class JsonFriendlyRecordTest { public class JsonFriendlyRecordTest {
private KinesisClientRecord kinesisClientRecord; private KinesisClientRecord kinesisClientRecord;
@ -48,7 +47,7 @@ public class JsonFriendlyRecordTest {
@Test @Test
public void testRecordHandlesNoByteArrayBuffer() { public void testRecordHandlesNoByteArrayBuffer() {
byte[] expected = new byte[] { 1, 2, 3, 4 }; byte[] expected = new byte[] {1, 2, 3, 4};
ByteBuffer expectedBuffer = ByteBuffer.allocateDirect(expected.length); ByteBuffer expectedBuffer = ByteBuffer.allocateDirect(expected.length);
@ -64,7 +63,7 @@ public class JsonFriendlyRecordTest {
@Test @Test
public void testRecordHandlesArrayByteBuffer() { public void testRecordHandlesArrayByteBuffer() {
ByteBuffer expected = ByteBuffer.wrap(new byte[] { 1, 2, 3, 4 }); ByteBuffer expected = ByteBuffer.wrap(new byte[] {1, 2, 3, 4});
kinesisClientRecord = defaultRecord().data(expected).build(); kinesisClientRecord = defaultRecord().data(expected).build();
JsonFriendlyRecord jsonFriendlyRecord = JsonFriendlyRecord.fromKinesisClientRecord(kinesisClientRecord); JsonFriendlyRecord jsonFriendlyRecord = JsonFriendlyRecord.fromKinesisClientRecord(kinesisClientRecord);
@ -82,14 +81,15 @@ public class JsonFriendlyRecordTest {
private RecordMatcher(KinesisClientRecord expected) { private RecordMatcher(KinesisClientRecord expected) {
this.matchers = Arrays.asList( this.matchers = Arrays.asList(
new FieldMatcher<>("approximateArrivalTimestamp", new FieldMatcher<>(
"approximateArrivalTimestamp",
equalTo(expected.approximateArrivalTimestamp().toEpochMilli()), equalTo(expected.approximateArrivalTimestamp().toEpochMilli()),
JsonFriendlyRecord::getApproximateArrivalTimestamp), JsonFriendlyRecord::getApproximateArrivalTimestamp),
new FieldMatcher<>("partitionKey", expected::partitionKey, JsonFriendlyRecord::getPartitionKey), new FieldMatcher<>("partitionKey", expected::partitionKey, JsonFriendlyRecord::getPartitionKey),
new FieldMatcher<>("sequenceNumber", expected::sequenceNumber, new FieldMatcher<>(
JsonFriendlyRecord::getSequenceNumber), "sequenceNumber", expected::sequenceNumber, JsonFriendlyRecord::getSequenceNumber),
new FieldMatcher<>("subSequenceNumber", expected::subSequenceNumber, new FieldMatcher<>(
JsonFriendlyRecord::getSubSequenceNumber), "subSequenceNumber", expected::subSequenceNumber, JsonFriendlyRecord::getSubSequenceNumber),
new FieldMatcher<>("data", dataEquivalentTo(expected.data()), JsonFriendlyRecord::getData)); new FieldMatcher<>("data", dataEquivalentTo(expected.data()), JsonFriendlyRecord::getData));
this.expected = expected; this.expected = expected;
@ -97,13 +97,16 @@ public class JsonFriendlyRecordTest {
@Override @Override
protected boolean matchesSafely(JsonFriendlyRecord item, Description mismatchDescription) { protected boolean matchesSafely(JsonFriendlyRecord item, Description mismatchDescription) {
return matchers.stream().map(m -> { return matchers.stream()
if (!m.matches(item)) { .map(m -> {
m.describeMismatch(item, mismatchDescription); if (!m.matches(item)) {
return false; m.describeMismatch(item, mismatchDescription);
} return false;
return true; }
}).reduce((l, r) -> l && r).orElse(true); return true;
})
.reduce((l, r) -> l && r)
.orElse(true);
} }
@Override @Override
@ -160,8 +163,9 @@ public class JsonFriendlyRecordTest {
} }
private KinesisClientRecord.KinesisClientRecordBuilder defaultRecord() { private KinesisClientRecord.KinesisClientRecordBuilder defaultRecord() {
return KinesisClientRecord.builder().partitionKey("test-partition").sequenceNumber("123") return KinesisClientRecord.builder()
.partitionKey("test-partition")
.sequenceNumber("123")
.approximateArrivalTimestamp(Instant.now()); .approximateArrivalTimestamp(Instant.now());
} }
}
}

View file

@ -17,66 +17,65 @@ package software.amazon.kinesis.multilang.messages;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.util.Collections; import java.util.Collections;
import org.junit.Assert;
import org.junit.Test;
import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.Assert;
import org.junit.Test;
import software.amazon.kinesis.lifecycle.ShutdownReason;
import software.amazon.kinesis.lifecycle.events.InitializationInput; import software.amazon.kinesis.lifecycle.events.InitializationInput;
import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput; import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput;
import software.amazon.kinesis.lifecycle.ShutdownReason;
import software.amazon.kinesis.retrieval.KinesisClientRecord; import software.amazon.kinesis.retrieval.KinesisClientRecord;
public class MessageTest { public class MessageTest {
@Test @Test
public void toStringTest() { public void toStringTest() {
Message[] messages = new Message[]{ Message[] messages = new Message[] {
new CheckpointMessage("1234567890", 0L, null), new CheckpointMessage("1234567890", 0L, null),
new InitializeMessage(InitializationInput.builder().shardId("shard-123").build()), new InitializeMessage(
new ProcessRecordsMessage(ProcessRecordsInput.builder() InitializationInput.builder().shardId("shard-123").build()),
.records(Collections.singletonList( new ProcessRecordsMessage(ProcessRecordsInput.builder()
KinesisClientRecord.builder() .records(Collections.singletonList(KinesisClientRecord.builder()
.data(ByteBuffer.wrap("cat".getBytes())) .data(ByteBuffer.wrap("cat".getBytes()))
.partitionKey("cat") .partitionKey("cat")
.sequenceNumber("555") .sequenceNumber("555")
.build())) .build()))
.build()), .build()),
new ShutdownMessage(ShutdownReason.LEASE_LOST), new ShutdownMessage(ShutdownReason.LEASE_LOST),
new StatusMessage("processRecords"), new StatusMessage("processRecords"),
new InitializeMessage(), new InitializeMessage(),
new ProcessRecordsMessage(), new ProcessRecordsMessage(),
new ShutdownRequestedMessage(), new ShutdownRequestedMessage(),
new LeaseLostMessage(), new LeaseLostMessage(),
new ShardEndedMessage(), new ShardEndedMessage(),
}; };
// TODO: fix this // TODO: fix this
for (int i = 0; i < messages.length; i++) { for (int i = 0; i < messages.length; i++) {
System.out.println(messages[i].toString()); System.out.println(messages[i].toString());
Assert.assertTrue("Each message should contain the action field", messages[i].toString().contains("action")); Assert.assertTrue(
"Each message should contain the action field",
messages[i].toString().contains("action"));
} }
// Hit this constructor // Hit this constructor
KinesisClientRecord defaultJsonFriendlyRecord = KinesisClientRecord.builder().build(); KinesisClientRecord defaultJsonFriendlyRecord =
KinesisClientRecord.builder().build();
Assert.assertNull(defaultJsonFriendlyRecord.partitionKey()); Assert.assertNull(defaultJsonFriendlyRecord.partitionKey());
Assert.assertNull(defaultJsonFriendlyRecord.data()); Assert.assertNull(defaultJsonFriendlyRecord.data());
Assert.assertNull(defaultJsonFriendlyRecord.sequenceNumber()); Assert.assertNull(defaultJsonFriendlyRecord.sequenceNumber());
Assert.assertNull(new ShutdownMessage(null).getReason()); Assert.assertNull(new ShutdownMessage(null).getReason());
// Hit the bad object mapping path // Hit the bad object mapping path
Message withBadMapper = new Message() { Message withBadMapper = new Message() {}.withObjectMapper(new ObjectMapper() {
}.withObjectMapper(new ObjectMapper() {
/** /**
* *
*/ */
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
@Override @Override
public String writeValueAsString(Object m) throws JsonProcessingException { public String writeValueAsString(Object m) throws JsonProcessingException {
throw new JsonProcessingException(new Throwable()) { throw new JsonProcessingException(new Throwable()) {};
};
} }
}); });
String s = withBadMapper.toString(); String s = withBadMapper.toString();

View file

@ -22,5 +22,4 @@ import java.lang.annotation.RetentionPolicy;
* Any class/method/variable marked with this annotation is subject to breaking changes between minor releases. * Any class/method/variable marked with this annotation is subject to breaking changes between minor releases.
*/ */
@Retention(RetentionPolicy.CLASS) @Retention(RetentionPolicy.CLASS)
public @interface KinesisClientInternalApi { public @interface KinesisClientInternalApi {}
}

View file

@ -40,7 +40,10 @@ public class Checkpoint {
* @param pendingCheckpoint the pending checkpoint sequence number - can be null. * @param pendingCheckpoint the pending checkpoint sequence number - can be null.
* @param pendingCheckpointState the pending checkpoint state - can be null. * @param pendingCheckpointState the pending checkpoint state - can be null.
*/ */
public Checkpoint(final ExtendedSequenceNumber checkpoint, final ExtendedSequenceNumber pendingCheckpoint, byte[] pendingCheckpointState) { public Checkpoint(
final ExtendedSequenceNumber checkpoint,
final ExtendedSequenceNumber pendingCheckpoint,
byte[] pendingCheckpointState) {
if (checkpoint == null || checkpoint.sequenceNumber().isEmpty()) { if (checkpoint == null || checkpoint.sequenceNumber().isEmpty()) {
throw new IllegalArgumentException("Checkpoint cannot be null or empty"); throw new IllegalArgumentException("Checkpoint cannot be null or empty");
} }

View file

@ -15,7 +15,6 @@
package software.amazon.kinesis.checkpoint; package software.amazon.kinesis.checkpoint;
import lombok.Data; import lombok.Data;
import lombok.experimental.Accessors; import lombok.experimental.Accessors;
import software.amazon.kinesis.checkpoint.dynamodb.DynamoDBCheckpointFactory; import software.amazon.kinesis.checkpoint.dynamodb.DynamoDBCheckpointFactory;

View file

@ -60,9 +60,7 @@ public class DoesNothingPreparedCheckpointer implements PreparedCheckpointer {
@Override @Override
public void checkpoint() public void checkpoint()
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException, throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
IllegalArgumentException { IllegalArgumentException {
// This method does nothing // This method does nothing
} }
} }

View file

@ -20,10 +20,9 @@ import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
import org.apache.commons.lang3.StringUtils;
import lombok.Data; import lombok.Data;
import lombok.experimental.Accessors; import lombok.experimental.Accessors;
import org.apache.commons.lang3.StringUtils;
/** /**
* This supports extracting the shardId from a sequence number. * This supports extracting the shardId from a sequence number.
@ -98,11 +97,15 @@ public class SequenceNumberValidator {
} }
} }
private static final List<SequenceNumberReader> SEQUENCE_NUMBER_READERS = Collections private static final List<SequenceNumberReader> SEQUENCE_NUMBER_READERS =
.singletonList(new V2SequenceNumberReader()); Collections.singletonList(new V2SequenceNumberReader());
private Optional<SequenceNumberComponents> retrieveComponentsFor(String sequenceNumber) { private Optional<SequenceNumberComponents> retrieveComponentsFor(String sequenceNumber) {
return SEQUENCE_NUMBER_READERS.stream().map(r -> r.read(sequenceNumber)).filter(Optional::isPresent).map(Optional::get).findFirst(); return SEQUENCE_NUMBER_READERS.stream()
.map(r -> r.read(sequenceNumber))
.filter(Optional::isPresent)
.map(Optional::get)
.findFirst();
} }
/** /**
@ -118,7 +121,7 @@ public class SequenceNumberValidator {
* </ul> * </ul>
* </strong> * </strong>
* </p> * </p>
* *
* @param sequenceNumber * @param sequenceNumber
* the sequence number to extract the version from * the sequence number to extract the version from
* @return an Optional containing the version if a compatible sequence number reader can be found, an empty Optional * @return an Optional containing the version if a compatible sequence number reader can be found, an empty Optional
@ -184,5 +187,4 @@ public class SequenceNumberValidator {
public Optional<Boolean> validateSequenceNumberForShard(String sequenceNumber, String shardId) { public Optional<Boolean> validateSequenceNumberForShard(String sequenceNumber, String shardId) {
return shardIdFor(sequenceNumber).map(s -> StringUtils.equalsIgnoreCase(s, shardId)); return shardIdFor(sequenceNumber).map(s -> StringUtils.equalsIgnoreCase(s, shardId));
} }
} }

View file

@ -38,8 +38,8 @@ public class ShardPreparedCheckpointer implements PreparedCheckpointer {
* @param pendingCheckpointSequenceNumber sequence number to checkpoint at * @param pendingCheckpointSequenceNumber sequence number to checkpoint at
* @param checkpointer checkpointer to use * @param checkpointer checkpointer to use
*/ */
public ShardPreparedCheckpointer(ExtendedSequenceNumber pendingCheckpointSequenceNumber, public ShardPreparedCheckpointer(
RecordProcessorCheckpointer checkpointer) { ExtendedSequenceNumber pendingCheckpointSequenceNumber, RecordProcessorCheckpointer checkpointer) {
this.pendingCheckpointSequenceNumber = pendingCheckpointSequenceNumber; this.pendingCheckpointSequenceNumber = pendingCheckpointSequenceNumber;
this.checkpointer = checkpointer; this.checkpointer = checkpointer;
} }
@ -58,8 +58,8 @@ public class ShardPreparedCheckpointer implements PreparedCheckpointer {
@Override @Override
public void checkpoint() public void checkpoint()
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException, throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
IllegalArgumentException { IllegalArgumentException {
checkpointer.checkpoint(pendingCheckpointSequenceNumber.sequenceNumber(), checkpointer.checkpoint(
pendingCheckpointSequenceNumber.subSequenceNumber()); pendingCheckpointSequenceNumber.sequenceNumber(), pendingCheckpointSequenceNumber.subSequenceNumber());
} }
} }

View file

@ -41,16 +41,22 @@ import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber;
public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpointer { public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpointer {
@NonNull @NonNull
private final ShardInfo shardInfo; private final ShardInfo shardInfo;
@NonNull @NonNull
@Getter @Accessors(fluent = true) @Getter
@Accessors(fluent = true)
private final Checkpointer checkpointer; private final Checkpointer checkpointer;
// Set to the last value set via checkpoint(). // Set to the last value set via checkpoint().
// Sample use: verify application shutdown() invoked checkpoint() at the end of a shard. // Sample use: verify application shutdown() invoked checkpoint() at the end of a shard.
@Getter @Accessors(fluent = true) @Getter
@Accessors(fluent = true)
private ExtendedSequenceNumber lastCheckpointValue; private ExtendedSequenceNumber lastCheckpointValue;
@Getter @Accessors(fluent = true)
@Getter
@Accessors(fluent = true)
private ExtendedSequenceNumber largestPermittedCheckpointValue; private ExtendedSequenceNumber largestPermittedCheckpointValue;
private ExtendedSequenceNumber sequenceNumberAtShardEnd; private ExtendedSequenceNumber sequenceNumberAtShardEnd;
/** /**
@ -60,8 +66,11 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
public synchronized void checkpoint() public synchronized void checkpoint()
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException { throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
if (log.isDebugEnabled()) { if (log.isDebugEnabled()) {
log.debug("Checkpointing {}, token {} at largest permitted value {}", ShardInfo.getLeaseKey(shardInfo), log.debug(
shardInfo.concurrencyToken(), this.largestPermittedCheckpointValue); "Checkpointing {}, token {} at largest permitted value {}",
ShardInfo.getLeaseKey(shardInfo),
shardInfo.concurrencyToken(),
this.largestPermittedCheckpointValue);
} }
advancePosition(this.largestPermittedCheckpointValue); advancePosition(this.largestPermittedCheckpointValue);
} }
@ -71,15 +80,15 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
*/ */
@Override @Override
public synchronized void checkpoint(Record record) public synchronized void checkpoint(Record record)
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException, throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
IllegalArgumentException { IllegalArgumentException {
// TODO: UserRecord Deprecation // TODO: UserRecord Deprecation
if (record == null) { if (record == null) {
throw new IllegalArgumentException("Could not checkpoint a null record"); throw new IllegalArgumentException("Could not checkpoint a null record");
} /* else if (record instanceof UserRecord) { } /* else if (record instanceof UserRecord) {
checkpoint(record.sequenceNumber(), ((UserRecord) record).subSequenceNumber()); checkpoint(record.sequenceNumber(), ((UserRecord) record).subSequenceNumber());
} */ else { } */ else {
checkpoint(record.sequenceNumber(), 0); checkpoint(record.sequenceNumber(), 0);
} }
} }
@ -89,8 +98,8 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
*/ */
@Override @Override
public synchronized void checkpoint(String sequenceNumber) public synchronized void checkpoint(String sequenceNumber)
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException, throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
IllegalArgumentException { IllegalArgumentException {
checkpoint(sequenceNumber, 0); checkpoint(sequenceNumber, 0);
} }
@ -99,12 +108,12 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
*/ */
@Override @Override
public synchronized void checkpoint(String sequenceNumber, long subSequenceNumber) public synchronized void checkpoint(String sequenceNumber, long subSequenceNumber)
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException, throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
IllegalArgumentException { IllegalArgumentException {
if (subSequenceNumber < 0) { if (subSequenceNumber < 0) {
throw new IllegalArgumentException("Could not checkpoint at invalid, negative subsequence number " throw new IllegalArgumentException(
+ subSequenceNumber); "Could not checkpoint at invalid, negative subsequence number " + subSequenceNumber);
} }
/* /*
@ -116,15 +125,18 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
&& newCheckpoint.compareTo(largestPermittedCheckpointValue) <= 0) { && newCheckpoint.compareTo(largestPermittedCheckpointValue) <= 0) {
if (log.isDebugEnabled()) { if (log.isDebugEnabled()) {
log.debug("Checkpointing {}, token {} at specific extended sequence number {}", ShardInfo.getLeaseKey(shardInfo), log.debug(
shardInfo.concurrencyToken(), newCheckpoint); "Checkpointing {}, token {} at specific extended sequence number {}",
ShardInfo.getLeaseKey(shardInfo),
shardInfo.concurrencyToken(),
newCheckpoint);
} }
this.advancePosition(newCheckpoint); this.advancePosition(newCheckpoint);
} else { } else {
throw new IllegalArgumentException(String.format( throw new IllegalArgumentException(String.format(
"Could not checkpoint at extended sequence number %s as it did not fall into acceptable range " "Could not checkpoint at extended sequence number %s as it did not fall into acceptable range "
+ "between the last checkpoint %s and the greatest extended sequence number passed to this " + "between the last checkpoint %s and the greatest extended sequence number passed to this "
+ "record processor %s", + "record processor %s",
newCheckpoint, this.lastCheckpointValue, this.largestPermittedCheckpointValue)); newCheckpoint, this.lastCheckpointValue, this.largestPermittedCheckpointValue));
} }
} }
@ -161,8 +173,8 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
if (record == null) { if (record == null) {
throw new IllegalArgumentException("Could not prepare checkpoint a null record"); throw new IllegalArgumentException("Could not prepare checkpoint a null record");
} /*else if (record instanceof UserRecord) { } /*else if (record instanceof UserRecord) {
return prepareCheckpoint(record.sequenceNumber(), ((UserRecord) record).subSequenceNumber()); return prepareCheckpoint(record.sequenceNumber(), ((UserRecord) record).subSequenceNumber());
} */ else { } */ else {
return prepareCheckpoint(record.sequenceNumber(), 0, applicationState); return prepareCheckpoint(record.sequenceNumber(), 0, applicationState);
} }
} }
@ -190,7 +202,8 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
*/ */
@Override @Override
public PreparedCheckpointer prepareCheckpoint(String sequenceNumber, byte[] applicationState) public PreparedCheckpointer prepareCheckpoint(String sequenceNumber, byte[] applicationState)
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException, IllegalArgumentException { throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
IllegalArgumentException {
return prepareCheckpoint(sequenceNumber, 0, applicationState); return prepareCheckpoint(sequenceNumber, 0, applicationState);
} }
@ -207,11 +220,13 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
* {@inheritDoc} * {@inheritDoc}
*/ */
@Override @Override
public PreparedCheckpointer prepareCheckpoint(String sequenceNumber, long subSequenceNumber, byte[] applicationState) public PreparedCheckpointer prepareCheckpoint(
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException, IllegalArgumentException { String sequenceNumber, long subSequenceNumber, byte[] applicationState)
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
IllegalArgumentException {
if (subSequenceNumber < 0) { if (subSequenceNumber < 0) {
throw new IllegalArgumentException("Could not checkpoint at invalid, negative subsequence number " throw new IllegalArgumentException(
+ subSequenceNumber); "Could not checkpoint at invalid, negative subsequence number " + subSequenceNumber);
} }
/* /*
@ -223,8 +238,11 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
&& pendingCheckpoint.compareTo(largestPermittedCheckpointValue) <= 0) { && pendingCheckpoint.compareTo(largestPermittedCheckpointValue) <= 0) {
if (log.isDebugEnabled()) { if (log.isDebugEnabled()) {
log.debug("Preparing checkpoint {}, token {} at specific extended sequence number {}", log.debug(
ShardInfo.getLeaseKey(shardInfo), shardInfo.concurrencyToken(), pendingCheckpoint); "Preparing checkpoint {}, token {} at specific extended sequence number {}",
ShardInfo.getLeaseKey(shardInfo),
shardInfo.concurrencyToken(),
pendingCheckpoint);
} }
return doPrepareCheckpoint(pendingCheckpoint, applicationState); return doPrepareCheckpoint(pendingCheckpoint, applicationState);
} else { } else {
@ -258,7 +276,6 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
this.sequenceNumberAtShardEnd = extendedSequenceNumber; this.sequenceNumberAtShardEnd = extendedSequenceNumber;
} }
/** /**
* Internal API - has package level access only for testing purposes. * Internal API - has package level access only for testing purposes.
* *
@ -270,29 +287,35 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
* @throws InvalidStateException * @throws InvalidStateException
*/ */
void advancePosition(String sequenceNumber) void advancePosition(String sequenceNumber)
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException { throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
advancePosition(new ExtendedSequenceNumber(sequenceNumber)); advancePosition(new ExtendedSequenceNumber(sequenceNumber));
} }
void advancePosition(ExtendedSequenceNumber extendedSequenceNumber) void advancePosition(ExtendedSequenceNumber extendedSequenceNumber)
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException { throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
ExtendedSequenceNumber checkpointToRecord = extendedSequenceNumber; ExtendedSequenceNumber checkpointToRecord = extendedSequenceNumber;
if (sequenceNumberAtShardEnd != null && sequenceNumberAtShardEnd.equals(extendedSequenceNumber)) { if (sequenceNumberAtShardEnd != null && sequenceNumberAtShardEnd.equals(extendedSequenceNumber)) {
// If we are about to checkpoint the very last sequence number for this shard, we might as well // If we are about to checkpoint the very last sequence number for this shard, we might as well
// just checkpoint at SHARD_END // just checkpoint at SHARD_END
checkpointToRecord = ExtendedSequenceNumber.SHARD_END; checkpointToRecord = ExtendedSequenceNumber.SHARD_END;
} }
// Don't checkpoint a value we already successfully checkpointed // Don't checkpoint a value we already successfully checkpointed
if (extendedSequenceNumber != null && !extendedSequenceNumber.equals(lastCheckpointValue)) { if (extendedSequenceNumber != null && !extendedSequenceNumber.equals(lastCheckpointValue)) {
try { try {
if (log.isDebugEnabled()) { if (log.isDebugEnabled()) {
log.debug("Setting {}, token {} checkpoint to {}", ShardInfo.getLeaseKey(shardInfo), log.debug(
shardInfo.concurrencyToken(), checkpointToRecord); "Setting {}, token {} checkpoint to {}",
ShardInfo.getLeaseKey(shardInfo),
shardInfo.concurrencyToken(),
checkpointToRecord);
} }
checkpointer.setCheckpoint(ShardInfo.getLeaseKey(shardInfo), checkpointToRecord, shardInfo.concurrencyToken()); checkpointer.setCheckpoint(
ShardInfo.getLeaseKey(shardInfo), checkpointToRecord, shardInfo.concurrencyToken());
lastCheckpointValue = checkpointToRecord; lastCheckpointValue = checkpointToRecord;
} catch (ThrottlingException | ShutdownException | InvalidStateException } catch (ThrottlingException
| ShutdownException
| InvalidStateException
| KinesisClientLibDependencyException e) { | KinesisClientLibDependencyException e) {
throw e; throw e;
} catch (KinesisClientLibException e) { } catch (KinesisClientLibException e) {
@ -325,7 +348,8 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
* @throws ThrottlingException * @throws ThrottlingException
* @throws ShutdownException * @throws ShutdownException
*/ */
private PreparedCheckpointer doPrepareCheckpoint(ExtendedSequenceNumber extendedSequenceNumber, byte[] applicationState) private PreparedCheckpointer doPrepareCheckpoint(
ExtendedSequenceNumber extendedSequenceNumber, byte[] applicationState)
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException { throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
ExtendedSequenceNumber newPrepareCheckpoint = extendedSequenceNumber; ExtendedSequenceNumber newPrepareCheckpoint = extendedSequenceNumber;
@ -343,8 +367,14 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
} }
try { try {
checkpointer.prepareCheckpoint(ShardInfo.getLeaseKey(shardInfo), newPrepareCheckpoint, shardInfo.concurrencyToken(), applicationState); checkpointer.prepareCheckpoint(
} catch (ThrottlingException | ShutdownException | InvalidStateException ShardInfo.getLeaseKey(shardInfo),
newPrepareCheckpoint,
shardInfo.concurrencyToken(),
applicationState);
} catch (ThrottlingException
| ShutdownException
| InvalidStateException
| KinesisClientLibDependencyException e) { | KinesisClientLibDependencyException e) {
throw e; throw e;
} catch (KinesisClientLibException e) { } catch (KinesisClientLibException e) {

View file

@ -29,9 +29,8 @@ import software.amazon.kinesis.processor.Checkpointer;
@KinesisClientInternalApi @KinesisClientInternalApi
public class DynamoDBCheckpointFactory implements CheckpointFactory { public class DynamoDBCheckpointFactory implements CheckpointFactory {
@Override @Override
public Checkpointer createCheckpointer(final LeaseCoordinator leaseLeaseCoordinator, public Checkpointer createCheckpointer(
final LeaseRefresher leaseRefresher) { final LeaseCoordinator leaseLeaseCoordinator, final LeaseRefresher leaseRefresher) {
return new DynamoDBCheckpointer(leaseLeaseCoordinator, leaseRefresher); return new DynamoDBCheckpointer(leaseLeaseCoordinator, leaseRefresher);
} }
} }

View file

@ -19,7 +19,6 @@ import java.util.Objects;
import java.util.UUID; import java.util.UUID;
import com.google.common.annotations.VisibleForTesting; import com.google.common.annotations.VisibleForTesting;
import lombok.NonNull; import lombok.NonNull;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -48,14 +47,16 @@ import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber;
public class DynamoDBCheckpointer implements Checkpointer { public class DynamoDBCheckpointer implements Checkpointer {
@NonNull @NonNull
private final LeaseCoordinator leaseCoordinator; private final LeaseCoordinator leaseCoordinator;
@NonNull @NonNull
private final LeaseRefresher leaseRefresher; private final LeaseRefresher leaseRefresher;
private String operation; private String operation;
@Override @Override
public void setCheckpoint(final String leaseKey, final ExtendedSequenceNumber checkpointValue, public void setCheckpoint(
final String concurrencyToken) throws KinesisClientLibException { final String leaseKey, final ExtendedSequenceNumber checkpointValue, final String concurrencyToken)
throws KinesisClientLibException {
try { try {
boolean wasSuccessful = setCheckpoint(leaseKey, checkpointValue, UUID.fromString(concurrencyToken)); boolean wasSuccessful = setCheckpoint(leaseKey, checkpointValue, UUID.fromString(concurrencyToken));
if (!wasSuccessful) { if (!wasSuccessful) {
@ -97,17 +98,22 @@ public class DynamoDBCheckpointer implements Checkpointer {
} }
@Override @Override
public void prepareCheckpoint(final String leaseKey, final ExtendedSequenceNumber pendingCheckpoint, public void prepareCheckpoint(
final String concurrencyToken) throws KinesisClientLibException { final String leaseKey, final ExtendedSequenceNumber pendingCheckpoint, final String concurrencyToken)
throws KinesisClientLibException {
prepareCheckpoint(leaseKey, pendingCheckpoint, concurrencyToken, null); prepareCheckpoint(leaseKey, pendingCheckpoint, concurrencyToken, null);
} }
@Override @Override
public void prepareCheckpoint(String leaseKey, ExtendedSequenceNumber pendingCheckpoint, String concurrencyToken, public void prepareCheckpoint(
byte[] pendingCheckpointState) throws KinesisClientLibException { String leaseKey,
ExtendedSequenceNumber pendingCheckpoint,
String concurrencyToken,
byte[] pendingCheckpointState)
throws KinesisClientLibException {
try { try {
boolean wasSuccessful = boolean wasSuccessful = prepareCheckpoint(
prepareCheckpoint(leaseKey, pendingCheckpoint, UUID.fromString(concurrencyToken), pendingCheckpointState); leaseKey, pendingCheckpoint, UUID.fromString(concurrencyToken), pendingCheckpointState);
if (!wasSuccessful) { if (!wasSuccessful) {
throw new ShutdownException( throw new ShutdownException(
"Can't prepare checkpoint - instance doesn't hold the lease for this shard"); "Can't prepare checkpoint - instance doesn't hold the lease for this shard");
@ -128,8 +134,10 @@ public class DynamoDBCheckpointer implements Checkpointer {
throws DependencyException, InvalidStateException, ProvisionedThroughputException { throws DependencyException, InvalidStateException, ProvisionedThroughputException {
Lease lease = leaseCoordinator.getCurrentlyHeldLease(leaseKey); Lease lease = leaseCoordinator.getCurrentlyHeldLease(leaseKey);
if (lease == null) { if (lease == null) {
log.info("Worker {} could not update checkpoint for shard {} because it does not hold the lease", log.info(
leaseCoordinator.workerIdentifier(), leaseKey); "Worker {} could not update checkpoint for shard {} because it does not hold the lease",
leaseCoordinator.workerIdentifier(),
leaseKey);
return false; return false;
} }
@ -141,12 +149,18 @@ public class DynamoDBCheckpointer implements Checkpointer {
return leaseCoordinator.updateLease(lease, concurrencyToken, operation, leaseKey); return leaseCoordinator.updateLease(lease, concurrencyToken, operation, leaseKey);
} }
boolean prepareCheckpoint(String leaseKey, ExtendedSequenceNumber pendingCheckpoint, UUID concurrencyToken, byte[] pendingCheckpointState) boolean prepareCheckpoint(
String leaseKey,
ExtendedSequenceNumber pendingCheckpoint,
UUID concurrencyToken,
byte[] pendingCheckpointState)
throws DependencyException, InvalidStateException, ProvisionedThroughputException { throws DependencyException, InvalidStateException, ProvisionedThroughputException {
Lease lease = leaseCoordinator.getCurrentlyHeldLease(leaseKey); Lease lease = leaseCoordinator.getCurrentlyHeldLease(leaseKey);
if (lease == null) { if (lease == null) {
log.info("Worker {} could not prepare checkpoint for shard {} because it does not hold the lease", log.info(
leaseCoordinator.workerIdentifier(), leaseKey); "Worker {} could not prepare checkpoint for shard {} because it does not hold the lease",
leaseCoordinator.workerIdentifier(),
leaseKey);
return false; return false;
} }

View file

@ -15,10 +15,8 @@
package software.amazon.kinesis.common; package software.amazon.kinesis.common;
public class CommonCalculations { public class CommonCalculations {
/** /**
* Convenience method for calculating renewer intervals in milliseconds. * Convenience method for calculating renewer intervals in milliseconds.
* *

View file

@ -19,12 +19,11 @@ import java.util.function.Function;
import lombok.EqualsAndHashCode; import lombok.EqualsAndHashCode;
import lombok.Getter; import lombok.Getter;
import lombok.NonNull;
import lombok.Setter; import lombok.Setter;
import lombok.ToString; import lombok.ToString;
import org.apache.commons.lang3.StringUtils;
import lombok.NonNull;
import lombok.experimental.Accessors; import lombok.experimental.Accessors;
import org.apache.commons.lang3.StringUtils;
import software.amazon.awssdk.arns.Arn; import software.amazon.awssdk.arns.Arn;
import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient; import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient;
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient; import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
@ -35,9 +34,9 @@ import software.amazon.kinesis.coordinator.CoordinatorConfig;
import software.amazon.kinesis.leases.LeaseManagementConfig; import software.amazon.kinesis.leases.LeaseManagementConfig;
import software.amazon.kinesis.lifecycle.LifecycleConfig; import software.amazon.kinesis.lifecycle.LifecycleConfig;
import software.amazon.kinesis.metrics.MetricsConfig; import software.amazon.kinesis.metrics.MetricsConfig;
import software.amazon.kinesis.processor.MultiStreamTracker;
import software.amazon.kinesis.processor.ProcessorConfig; import software.amazon.kinesis.processor.ProcessorConfig;
import software.amazon.kinesis.processor.ShardRecordProcessorFactory; import software.amazon.kinesis.processor.ShardRecordProcessorFactory;
import software.amazon.kinesis.processor.MultiStreamTracker;
import software.amazon.kinesis.processor.SingleStreamTracker; import software.amazon.kinesis.processor.SingleStreamTracker;
import software.amazon.kinesis.processor.StreamTracker; import software.amazon.kinesis.processor.StreamTracker;
import software.amazon.kinesis.retrieval.RetrievalConfig; import software.amazon.kinesis.retrieval.RetrievalConfig;
@ -45,7 +44,10 @@ import software.amazon.kinesis.retrieval.RetrievalConfig;
/** /**
* This Builder is useful to create all configurations for the KCL with default values. * This Builder is useful to create all configurations for the KCL with default values.
*/ */
@Getter @Setter @ToString @EqualsAndHashCode @Getter
@Setter
@ToString
@EqualsAndHashCode
@Accessors(fluent = true) @Accessors(fluent = true)
public class ConfigsBuilder { public class ConfigsBuilder {
/** /**
@ -139,11 +141,16 @@ public class ConfigsBuilder {
* @param workerIdentifier * @param workerIdentifier
* @param shardRecordProcessorFactory * @param shardRecordProcessorFactory
*/ */
public ConfigsBuilder(@NonNull String streamName, @NonNull String applicationName, public ConfigsBuilder(
@NonNull KinesisAsyncClient kinesisClient, @NonNull DynamoDbAsyncClient dynamoDBClient, @NonNull String streamName,
@NonNull CloudWatchAsyncClient cloudWatchClient, @NonNull String workerIdentifier, @NonNull String applicationName,
@NonNull KinesisAsyncClient kinesisClient,
@NonNull DynamoDbAsyncClient dynamoDBClient,
@NonNull CloudWatchAsyncClient cloudWatchClient,
@NonNull String workerIdentifier,
@NonNull ShardRecordProcessorFactory shardRecordProcessorFactory) { @NonNull ShardRecordProcessorFactory shardRecordProcessorFactory) {
this(new SingleStreamTracker(streamName), this(
new SingleStreamTracker(streamName),
applicationName, applicationName,
kinesisClient, kinesisClient,
dynamoDBClient, dynamoDBClient,
@ -163,11 +170,16 @@ public class ConfigsBuilder {
* @param workerIdentifier * @param workerIdentifier
* @param shardRecordProcessorFactory * @param shardRecordProcessorFactory
*/ */
public ConfigsBuilder(@NonNull Arn streamArn, @NonNull String applicationName, public ConfigsBuilder(
@NonNull KinesisAsyncClient kinesisClient, @NonNull DynamoDbAsyncClient dynamoDBClient, @NonNull Arn streamArn,
@NonNull CloudWatchAsyncClient cloudWatchClient, @NonNull String workerIdentifier, @NonNull String applicationName,
@NonNull ShardRecordProcessorFactory shardRecordProcessorFactory) { @NonNull KinesisAsyncClient kinesisClient,
this(new SingleStreamTracker(streamArn), @NonNull DynamoDbAsyncClient dynamoDBClient,
@NonNull CloudWatchAsyncClient cloudWatchClient,
@NonNull String workerIdentifier,
@NonNull ShardRecordProcessorFactory shardRecordProcessorFactory) {
this(
new SingleStreamTracker(streamArn),
applicationName, applicationName,
kinesisClient, kinesisClient,
dynamoDBClient, dynamoDBClient,
@ -187,9 +199,13 @@ public class ConfigsBuilder {
* @param workerIdentifier * @param workerIdentifier
* @param shardRecordProcessorFactory * @param shardRecordProcessorFactory
*/ */
public ConfigsBuilder(@NonNull StreamTracker streamTracker, @NonNull String applicationName, public ConfigsBuilder(
@NonNull KinesisAsyncClient kinesisClient, @NonNull DynamoDbAsyncClient dynamoDBClient, @NonNull StreamTracker streamTracker,
@NonNull CloudWatchAsyncClient cloudWatchClient, @NonNull String workerIdentifier, @NonNull String applicationName,
@NonNull KinesisAsyncClient kinesisClient,
@NonNull DynamoDbAsyncClient dynamoDBClient,
@NonNull CloudWatchAsyncClient cloudWatchClient,
@NonNull String workerIdentifier,
@NonNull ShardRecordProcessorFactory shardRecordProcessorFactory) { @NonNull ShardRecordProcessorFactory shardRecordProcessorFactory) {
this.applicationName = applicationName; this.applicationName = applicationName;
this.kinesisClient = kinesisClient; this.kinesisClient = kinesisClient;
@ -209,8 +225,11 @@ public class ConfigsBuilder {
public void streamTracker(StreamTracker streamTracker) { public void streamTracker(StreamTracker streamTracker) {
this.streamTracker = streamTracker; this.streamTracker = streamTracker;
this.appStreamTracker = DeprecationUtils.convert(streamTracker, this.appStreamTracker = DeprecationUtils.convert(streamTracker, singleStreamTracker -> singleStreamTracker
singleStreamTracker -> singleStreamTracker.streamConfigList().get(0).streamIdentifier().streamName()); .streamConfigList()
.get(0)
.streamIdentifier()
.streamName());
} }
/** /**

View file

@ -39,8 +39,7 @@ public final class DeprecationUtils {
*/ */
@Deprecated @Deprecated
public static <R> Either<MultiStreamTracker, R> convert( public static <R> Either<MultiStreamTracker, R> convert(
StreamTracker streamTracker, StreamTracker streamTracker, Function<SingleStreamTracker, R> converter) {
Function<SingleStreamTracker, R> converter) {
if (streamTracker instanceof MultiStreamTracker) { if (streamTracker instanceof MultiStreamTracker) {
return Either.left((MultiStreamTracker) streamTracker); return Either.left((MultiStreamTracker) streamTracker);
} else if (streamTracker instanceof SingleStreamTracker) { } else if (streamTracker instanceof SingleStreamTracker) {
@ -49,5 +48,4 @@ public final class DeprecationUtils {
throw new IllegalArgumentException("Unhandled StreamTracker: " + streamTracker); throw new IllegalArgumentException("Unhandled StreamTracker: " + streamTracker);
} }
} }
} }

View file

@ -15,12 +15,12 @@
package software.amazon.kinesis.common; package software.amazon.kinesis.common;
import org.slf4j.Logger;
import software.amazon.kinesis.annotations.KinesisClientInternalApi;
import java.time.Duration; import java.time.Duration;
import java.time.Instant; import java.time.Instant;
import org.slf4j.Logger;
import software.amazon.kinesis.annotations.KinesisClientInternalApi;
import static software.amazon.kinesis.lifecycle.ShardConsumer.MAX_TIME_BETWEEN_REQUEST_RESPONSE; import static software.amazon.kinesis.lifecycle.ShardConsumer.MAX_TIME_BETWEEN_REQUEST_RESPONSE;
@KinesisClientInternalApi @KinesisClientInternalApi
@ -32,18 +32,22 @@ public class DiagnosticUtils {
* @param enqueueTimestamp of the event submitted to the executor service * @param enqueueTimestamp of the event submitted to the executor service
* @param log Slf4j Logger from RecordPublisher to log the events * @param log Slf4j Logger from RecordPublisher to log the events
*/ */
public static void takeDelayedDeliveryActionIfRequired(String resourceIdentifier, Instant enqueueTimestamp, Logger log) { public static void takeDelayedDeliveryActionIfRequired(
final long durationBetweenEnqueueAndAckInMillis = Duration String resourceIdentifier, Instant enqueueTimestamp, Logger log) {
.between(enqueueTimestamp, Instant.now()).toMillis(); final long durationBetweenEnqueueAndAckInMillis =
Duration.between(enqueueTimestamp, Instant.now()).toMillis();
if (durationBetweenEnqueueAndAckInMillis > MAX_TIME_BETWEEN_REQUEST_RESPONSE / 3) { if (durationBetweenEnqueueAndAckInMillis > MAX_TIME_BETWEEN_REQUEST_RESPONSE / 3) {
// The above condition logs the warn msg if the delivery time exceeds 11 seconds. // The above condition logs the warn msg if the delivery time exceeds 11 seconds.
log.warn( log.warn(
"{}: Record delivery time to shard consumer is high at {} millis. Check the ExecutorStateEvent logs" "{}: Record delivery time to shard consumer is high at {} millis. Check the ExecutorStateEvent logs"
+ " to see the state of the executor service. Also check if the RecordProcessor's processing " + " to see the state of the executor service. Also check if the RecordProcessor's processing "
+ "time is high. ", + "time is high. ",
resourceIdentifier, durationBetweenEnqueueAndAckInMillis); resourceIdentifier,
durationBetweenEnqueueAndAckInMillis);
} else if (log.isDebugEnabled()) { } else if (log.isDebugEnabled()) {
log.debug("{}: Record delivery time to shard consumer is {} millis", resourceIdentifier, log.debug(
"{}: Record delivery time to shard consumer is {} millis",
resourceIdentifier,
durationBetweenEnqueueAndAckInMillis); durationBetweenEnqueueAndAckInMillis);
} }
} }

View file

@ -31,5 +31,4 @@ public class FutureUtils {
throw te; throw te;
} }
} }
} }

View file

@ -15,14 +15,14 @@
package software.amazon.kinesis.common; package software.amazon.kinesis.common;
import java.math.BigInteger;
import lombok.NonNull; import lombok.NonNull;
import lombok.Value; import lombok.Value;
import lombok.experimental.Accessors; import lombok.experimental.Accessors;
import org.apache.commons.lang3.Validate; import org.apache.commons.lang3.Validate;
import software.amazon.awssdk.services.kinesis.model.HashKeyRange; import software.amazon.awssdk.services.kinesis.model.HashKeyRange;
import java.math.BigInteger;
/** /**
* Lease POJO to hold the starting hashkey range and ending hashkey range of kinesis shards. * Lease POJO to hold the starting hashkey range and ending hashkey range of kinesis shards.
*/ */
@ -34,8 +34,11 @@ public class HashKeyRangeForLease {
private final BigInteger endingHashKey; private final BigInteger endingHashKey;
public HashKeyRangeForLease(BigInteger startingHashKey, BigInteger endingHashKey) { public HashKeyRangeForLease(BigInteger startingHashKey, BigInteger endingHashKey) {
Validate.isTrue(startingHashKey.compareTo(endingHashKey) < 0, Validate.isTrue(
"StartingHashKey %s must be less than EndingHashKey %s ", startingHashKey, endingHashKey); startingHashKey.compareTo(endingHashKey) < 0,
"StartingHashKey %s must be less than EndingHashKey %s ",
startingHashKey,
endingHashKey);
this.startingHashKey = startingHashKey; this.startingHashKey = startingHashKey;
this.endingHashKey = endingHashKey; this.endingHashKey = endingHashKey;
} }
@ -65,11 +68,15 @@ public class HashKeyRangeForLease {
* @param endingHashKeyStr * @param endingHashKeyStr
* @return HashKeyRangeForLease * @return HashKeyRangeForLease
*/ */
public static HashKeyRangeForLease deserialize(@NonNull String startingHashKeyStr, @NonNull String endingHashKeyStr) { public static HashKeyRangeForLease deserialize(
@NonNull String startingHashKeyStr, @NonNull String endingHashKeyStr) {
final BigInteger startingHashKey = new BigInteger(startingHashKeyStr); final BigInteger startingHashKey = new BigInteger(startingHashKeyStr);
final BigInteger endingHashKey = new BigInteger(endingHashKeyStr); final BigInteger endingHashKey = new BigInteger(endingHashKeyStr);
Validate.isTrue(startingHashKey.compareTo(endingHashKey) < 0, Validate.isTrue(
"StartingHashKey %s must be less than EndingHashKey %s ", startingHashKeyStr, endingHashKeyStr); startingHashKey.compareTo(endingHashKey) < 0,
"StartingHashKey %s must be less than EndingHashKey %s ",
startingHashKeyStr,
endingHashKeyStr);
return new HashKeyRangeForLease(startingHashKey, endingHashKey); return new HashKeyRangeForLease(startingHashKey, endingHashKey);
} }

View file

@ -14,16 +14,17 @@
*/ */
package software.amazon.kinesis.common; package software.amazon.kinesis.common;
import java.util.Date;
import lombok.EqualsAndHashCode; import lombok.EqualsAndHashCode;
import lombok.ToString; import lombok.ToString;
import java.util.Date;
/** /**
* Class that houses the entities needed to specify the position in the stream from where a new application should * Class that houses the entities needed to specify the position in the stream from where a new application should
* start. * start.
*/ */
@ToString @EqualsAndHashCode @ToString
@EqualsAndHashCode
public class InitialPositionInStreamExtended { public class InitialPositionInStreamExtended {
private final InitialPositionInStream position; private final InitialPositionInStream position;

View file

@ -15,14 +15,14 @@
package software.amazon.kinesis.common; package software.amazon.kinesis.common;
import java.time.Duration;
import software.amazon.awssdk.http.Protocol; import software.amazon.awssdk.http.Protocol;
import software.amazon.awssdk.http.nio.netty.Http2Configuration; import software.amazon.awssdk.http.nio.netty.Http2Configuration;
import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient; import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient;
import software.amazon.awssdk.services.kinesis.KinesisAsyncClient; import software.amazon.awssdk.services.kinesis.KinesisAsyncClient;
import software.amazon.awssdk.services.kinesis.KinesisAsyncClientBuilder; import software.amazon.awssdk.services.kinesis.KinesisAsyncClientBuilder;
import java.time.Duration;
/** /**
* Utility to setup KinesisAsyncClient to be used with KCL. * Utility to setup KinesisAsyncClient to be used with KCL.
*/ */
@ -42,9 +42,12 @@ public class KinesisClientUtil {
} }
public static KinesisAsyncClientBuilder adjustKinesisClientBuilder(KinesisAsyncClientBuilder builder) { public static KinesisAsyncClientBuilder adjustKinesisClientBuilder(KinesisAsyncClientBuilder builder) {
return builder.httpClientBuilder(NettyNioAsyncHttpClient.builder().maxConcurrency(Integer.MAX_VALUE) return builder.httpClientBuilder(NettyNioAsyncHttpClient.builder()
.http2Configuration(Http2Configuration.builder().initialWindowSize(INITIAL_WINDOW_SIZE_BYTES) .maxConcurrency(Integer.MAX_VALUE)
.healthCheckPingPeriod(Duration.ofMillis(HEALTH_CHECK_PING_PERIOD_MILLIS)).build()) .http2Configuration(Http2Configuration.builder()
.initialWindowSize(INITIAL_WINDOW_SIZE_BYTES)
.healthCheckPingPeriod(Duration.ofMillis(HEALTH_CHECK_PING_PERIOD_MILLIS))
.build())
.protocol(Protocol.HTTP2)); .protocol(Protocol.HTTP2));
} }
} }

View file

@ -63,12 +63,11 @@ public class KinesisRequestsBuilder {
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
private static <T extends AwsRequest.Builder> T appendUserAgent(final T builder) { private static <T extends AwsRequest.Builder> T appendUserAgent(final T builder) {
return (T) builder return (T) builder.overrideConfiguration(AwsRequestOverrideConfiguration.builder()
.overrideConfiguration( .addApiName(ApiName.builder()
AwsRequestOverrideConfiguration.builder() .name(RetrievalConfig.KINESIS_CLIENT_LIB_USER_AGENT)
.addApiName(ApiName.builder().name(RetrievalConfig.KINESIS_CLIENT_LIB_USER_AGENT) .version(RetrievalConfig.KINESIS_CLIENT_LIB_USER_AGENT_VERSION)
.version(RetrievalConfig.KINESIS_CLIENT_LIB_USER_AGENT_VERSION).build()) .build())
.build()); .build());
} }
} }

View file

@ -24,7 +24,7 @@ import lombok.experimental.Accessors;
*/ */
@Builder @Builder
@Getter @Getter
@Accessors(fluent=true) @Accessors(fluent = true)
public class LeaseCleanupConfig { public class LeaseCleanupConfig {
/** /**
* Interval at which to run lease cleanup thread. * Interval at which to run lease cleanup thread.

View file

@ -15,11 +15,11 @@
package software.amazon.kinesis.common; package software.amazon.kinesis.common;
import lombok.experimental.Accessors;
import java.util.Optional; import java.util.Optional;
@Accessors(fluent=true) import lombok.experimental.Accessors;
@Accessors(fluent = true)
public class RequestDetails { public class RequestDetails {
/** /**
@ -62,6 +62,4 @@ public class RequestDetails {
public String toString() { public String toString() {
return String.format("request id - %s, timestamp - %s", getRequestId(), getTimestamp()); return String.format("request id - %s, timestamp - %s", getRequestId(), getTimestamp());
} }
} }

View file

@ -28,8 +28,7 @@ import lombok.experimental.Accessors;
public class StreamConfig { public class StreamConfig {
@NonNull @NonNull
private final StreamIdentifier streamIdentifier; private final StreamIdentifier streamIdentifier;
private final InitialPositionInStreamExtended initialPositionInStreamExtended; private final InitialPositionInStreamExtended initialPositionInStreamExtended;
private String consumerArn; private String consumerArn;
} }

View file

@ -15,6 +15,10 @@
package software.amazon.kinesis.common; package software.amazon.kinesis.common;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.Builder; import lombok.Builder;
import lombok.EqualsAndHashCode; import lombok.EqualsAndHashCode;
@ -25,10 +29,6 @@ import software.amazon.awssdk.arns.Arn;
import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.utils.Validate; import software.amazon.awssdk.utils.Validate;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@Builder(access = AccessLevel.PRIVATE) @Builder(access = AccessLevel.PRIVATE)
@EqualsAndHashCode @EqualsAndHashCode
@Getter @Getter
@ -37,10 +37,13 @@ public class StreamIdentifier {
@Builder.Default @Builder.Default
private final Optional<String> accountIdOptional = Optional.empty(); private final Optional<String> accountIdOptional = Optional.empty();
@NonNull @NonNull
private final String streamName; private final String streamName;
@Builder.Default @Builder.Default
private final Optional<Long> streamCreationEpochOptional = Optional.empty(); private final Optional<Long> streamCreationEpochOptional = Optional.empty();
@Builder.Default @Builder.Default
@EqualsAndHashCode.Exclude @EqualsAndHashCode.Exclude
private final Optional<Arn> streamArnOptional = Optional.empty(); private final Optional<Arn> streamArnOptional = Optional.empty();
@ -49,8 +52,8 @@ public class StreamIdentifier {
* Pattern for a serialized {@link StreamIdentifier}. The valid format is * Pattern for a serialized {@link StreamIdentifier}. The valid format is
* {@code <accountId>:<streamName>:<creationEpoch>}. * {@code <accountId>:<streamName>:<creationEpoch>}.
*/ */
private static final Pattern STREAM_IDENTIFIER_PATTERN = Pattern.compile( private static final Pattern STREAM_IDENTIFIER_PATTERN =
"(?<accountId>[0-9]+):(?<streamName>[^:]+):(?<creationEpoch>[0-9]+)"); Pattern.compile("(?<accountId>[0-9]+):(?<streamName>[^:]+):(?<creationEpoch>[0-9]+)");
/** /**
* Pattern for a stream ARN. The valid format is * Pattern for a stream ARN. The valid format is
@ -74,8 +77,10 @@ public class StreamIdentifier {
final char delimiter = ':'; final char delimiter = ':';
final StringBuilder sb = new StringBuilder() final StringBuilder sb = new StringBuilder()
.append(accountIdOptional.get()).append(delimiter) .append(accountIdOptional.get())
.append(streamName).append(delimiter) .append(delimiter)
.append(streamName)
.append(delimiter)
.append(streamCreationEpochOptional.get()); .append(streamCreationEpochOptional.get());
return sb.toString(); return sb.toString();
} }
@ -146,9 +151,7 @@ public class StreamIdentifier {
public static StreamIdentifier singleStreamInstance(String streamName) { public static StreamIdentifier singleStreamInstance(String streamName) {
Validate.notEmpty(streamName, "StreamName should not be empty"); Validate.notEmpty(streamName, "StreamName should not be empty");
return StreamIdentifier.builder() return StreamIdentifier.builder().streamName(streamName).build();
.streamName(streamName)
.build();
} }
/** /**
@ -173,7 +176,8 @@ public class StreamIdentifier {
* @param streamArn * @param streamArn
*/ */
public static void validateArn(Arn streamArn) { public static void validateArn(Arn streamArn) {
if (!STREAM_ARN_PATTERN.matcher(streamArn.toString()).matches() || !streamArn.region().isPresent()) { if (!STREAM_ARN_PATTERN.matcher(streamArn.toString()).matches()
|| !streamArn.region().isPresent()) {
throw new IllegalArgumentException("Invalid streamArn " + streamArn); throw new IllegalArgumentException("Invalid streamArn " + streamArn);
} }
} }
@ -185,9 +189,7 @@ public class StreamIdentifier {
*/ */
private static void validateCreationEpoch(long creationEpoch) { private static void validateCreationEpoch(long creationEpoch) {
if (creationEpoch <= 0) { if (creationEpoch <= 0) {
throw new IllegalArgumentException( throw new IllegalArgumentException("Creation epoch must be > 0; received " + creationEpoch);
"Creation epoch must be > 0; received " + creationEpoch);
} }
} }
} }

View file

@ -96,5 +96,4 @@ public class CoordinatorConfig {
* <p>Default value: 1000L</p> * <p>Default value: 1000L</p>
*/ */
private long schedulerInitializationBackoffTimeMillis = 1000L; private long schedulerInitializationBackoffTimeMillis = 1000L;
} }

View file

@ -5,7 +5,6 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import software.amazon.kinesis.common.StreamIdentifier; import software.amazon.kinesis.common.StreamIdentifier;
/** /**

View file

@ -26,6 +26,7 @@ import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.BooleanSupplier; import java.util.function.BooleanSupplier;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import software.amazon.awssdk.utils.CollectionUtils; import software.amazon.awssdk.utils.CollectionUtils;
import software.amazon.kinesis.leases.Lease; import software.amazon.kinesis.leases.Lease;
@ -45,8 +46,7 @@ import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException;
* This ensures redundancy for shard-sync during host failures. * This ensures redundancy for shard-sync during host failures.
*/ */
@Slf4j @Slf4j
class DeterministicShuffleShardSyncLeaderDecider class DeterministicShuffleShardSyncLeaderDecider implements LeaderDecider {
implements LeaderDecider {
// Fixed seed so that the shuffle order is preserved across workers // Fixed seed so that the shuffle order is preserved across workers
static final int DETERMINISTIC_SHUFFLE_SEED = 1947; static final int DETERMINISTIC_SHUFFLE_SEED = 1947;
@ -67,13 +67,11 @@ class DeterministicShuffleShardSyncLeaderDecider
* @param leaderElectionThreadPool Thread-pool to be used for leaderElection. * @param leaderElectionThreadPool Thread-pool to be used for leaderElection.
* @param numPeriodicShardSyncWorkers Number of leaders that will be elected to perform periodic shard syncs. * @param numPeriodicShardSyncWorkers Number of leaders that will be elected to perform periodic shard syncs.
*/ */
DeterministicShuffleShardSyncLeaderDecider(LeaseRefresher leaseRefresher, DeterministicShuffleShardSyncLeaderDecider(
ScheduledExecutorService leaderElectionThreadPool, LeaseRefresher leaseRefresher,
int numPeriodicShardSyncWorkers) { ScheduledExecutorService leaderElectionThreadPool,
this(leaseRefresher, int numPeriodicShardSyncWorkers) {
leaderElectionThreadPool, this(leaseRefresher, leaderElectionThreadPool, numPeriodicShardSyncWorkers, new ReentrantReadWriteLock());
numPeriodicShardSyncWorkers,
new ReentrantReadWriteLock());
} }
/** /**
@ -82,10 +80,11 @@ class DeterministicShuffleShardSyncLeaderDecider
* @param numPeriodicShardSyncWorkers Number of leaders that will be elected to perform periodic shard syncs. * @param numPeriodicShardSyncWorkers Number of leaders that will be elected to perform periodic shard syncs.
* @param readWriteLock Mechanism to lock for reading and writing of critical components * @param readWriteLock Mechanism to lock for reading and writing of critical components
*/ */
DeterministicShuffleShardSyncLeaderDecider(LeaseRefresher leaseRefresher, DeterministicShuffleShardSyncLeaderDecider(
ScheduledExecutorService leaderElectionThreadPool, LeaseRefresher leaseRefresher,
int numPeriodicShardSyncWorkers, ScheduledExecutorService leaderElectionThreadPool,
ReadWriteLock readWriteLock) { int numPeriodicShardSyncWorkers,
ReadWriteLock readWriteLock) {
this.leaseRefresher = leaseRefresher; this.leaseRefresher = leaseRefresher;
this.leaderElectionThreadPool = leaderElectionThreadPool; this.leaderElectionThreadPool = leaderElectionThreadPool;
this.numPeriodicShardSyncWorkers = numPeriodicShardSyncWorkers; this.numPeriodicShardSyncWorkers = numPeriodicShardSyncWorkers;
@ -101,8 +100,12 @@ class DeterministicShuffleShardSyncLeaderDecider
try { try {
log.debug("Started leader election at: " + Instant.now()); log.debug("Started leader election at: " + Instant.now());
List<Lease> leases = leaseRefresher.listLeases(); List<Lease> leases = leaseRefresher.listLeases();
List<String> uniqueHosts = leases.stream().map(Lease::leaseOwner) List<String> uniqueHosts = leases.stream()
.filter(owner -> owner != null).distinct().sorted().collect(Collectors.toList()); .map(Lease::leaseOwner)
.filter(owner -> owner != null)
.distinct()
.sorted()
.collect(Collectors.toList());
Collections.shuffle(uniqueHosts, new Random(DETERMINISTIC_SHUFFLE_SEED)); Collections.shuffle(uniqueHosts, new Random(DETERMINISTIC_SHUFFLE_SEED));
int numShardSyncWorkers = Math.min(uniqueHosts.size(), numPeriodicShardSyncWorkers); int numShardSyncWorkers = Math.min(uniqueHosts.size(), numPeriodicShardSyncWorkers);
@ -137,8 +140,11 @@ class DeterministicShuffleShardSyncLeaderDecider
// The first run will be after a minute. // The first run will be after a minute.
// We don't need jitter since it is scheduled with a fixed delay and time taken to scan leases // We don't need jitter since it is scheduled with a fixed delay and time taken to scan leases
// will be different at different times and on different hosts/workers. // will be different at different times and on different hosts/workers.
leaderElectionThreadPool.scheduleWithFixedDelay(this::electLeaders, ELECTION_INITIAL_DELAY_MILLIS, leaderElectionThreadPool.scheduleWithFixedDelay(
ELECTION_SCHEDULING_INTERVAL_MILLIS, TimeUnit.MILLISECONDS); this::electLeaders,
ELECTION_INITIAL_DELAY_MILLIS,
ELECTION_SCHEDULING_INTERVAL_MILLIS,
TimeUnit.MILLISECONDS);
} }
return executeConditionCheckWithReadLock(() -> isWorkerLeaderForShardSync(workerId)); return executeConditionCheckWithReadLock(() -> isWorkerLeaderForShardSync(workerId));
@ -152,7 +158,8 @@ class DeterministicShuffleShardSyncLeaderDecider
log.info("Successfully stopped leader election on the worker"); log.info("Successfully stopped leader election on the worker");
} else { } else {
leaderElectionThreadPool.shutdownNow(); leaderElectionThreadPool.shutdownNow();
log.info(String.format("Stopped leader election thread after awaiting termination for %d milliseconds", log.info(String.format(
"Stopped leader election thread after awaiting termination for %d milliseconds",
AWAIT_TERMINATION_MILLIS)); AWAIT_TERMINATION_MILLIS));
} }

View file

@ -15,11 +15,11 @@
package software.amazon.kinesis.coordinator; package software.amazon.kinesis.coordinator;
import java.util.concurrent.ExecutorService;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
import software.amazon.kinesis.leases.LeaseCoordinator; import software.amazon.kinesis.leases.LeaseCoordinator;
import java.util.concurrent.ExecutorService;
/** /**
* Creates {@link DiagnosticEvent}s for logging and visibility * Creates {@link DiagnosticEvent}s for logging and visibility
*/ */

View file

@ -15,15 +15,15 @@
package software.amazon.kinesis.coordinator; package software.amazon.kinesis.coordinator;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadPoolExecutor;
import lombok.Getter; import lombok.Getter;
import lombok.ToString; import lombok.ToString;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import software.amazon.kinesis.annotations.KinesisClientInternalApi; import software.amazon.kinesis.annotations.KinesisClientInternalApi;
import software.amazon.kinesis.leases.LeaseCoordinator; import software.amazon.kinesis.leases.LeaseCoordinator;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadPoolExecutor;
@Getter @Getter
@ToString(exclude = "isThreadPoolExecutor") @ToString(exclude = "isThreadPoolExecutor")
@Slf4j @Slf4j

View file

@ -14,12 +14,12 @@
*/ */
package software.amazon.kinesis.coordinator; package software.amazon.kinesis.coordinator;
import java.util.concurrent.CountDownLatch;
import lombok.Builder; import lombok.Builder;
import lombok.Data; import lombok.Data;
import lombok.experimental.Accessors; import lombok.experimental.Accessors;
import java.util.concurrent.CountDownLatch;
@Data @Data
@Builder @Builder
@Accessors(fluent = true) @Accessors(fluent = true)

View file

@ -14,13 +14,12 @@
*/ */
package software.amazon.kinesis.coordinator; package software.amazon.kinesis.coordinator;
import lombok.extern.slf4j.Slf4j;
import java.util.concurrent.Callable; import java.util.concurrent.Callable;
import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
class GracefulShutdownCoordinator { class GracefulShutdownCoordinator {
/** /**
@ -31,8 +30,11 @@ class GracefulShutdownCoordinator {
CompletableFuture<Boolean> startGracefulShutdown(Callable<Boolean> shutdownCallable) { CompletableFuture<Boolean> startGracefulShutdown(Callable<Boolean> shutdownCallable) {
CompletableFuture<Boolean> cf = new CompletableFuture<>(); CompletableFuture<Boolean> cf = new CompletableFuture<>();
CompletableFuture.runAsync(() -> { CompletableFuture.runAsync(() -> {
try { cf.complete(shutdownCallable.call()); } try {
catch(Throwable ex) { cf.completeExceptionally(ex); } cf.complete(shutdownCallable.call());
} catch (Throwable ex) {
cf.completeExceptionally(ex);
}
}); });
return cf; return cf;
} }
@ -50,7 +52,8 @@ class GracefulShutdownCoordinator {
} }
private boolean isWorkerShutdownComplete(GracefulShutdownContext context) { private boolean isWorkerShutdownComplete(GracefulShutdownContext context) {
return context.scheduler().shutdownComplete() || context.scheduler().shardInfoShardConsumerMap().isEmpty(); return context.scheduler().shutdownComplete()
|| context.scheduler().shardInfoShardConsumerMap().isEmpty();
} }
private String awaitingLogMessage(GracefulShutdownContext context) { private String awaitingLogMessage(GracefulShutdownContext context) {
@ -92,12 +95,14 @@ class GracefulShutdownCoordinator {
throw new InterruptedException(); throw new InterruptedException();
} }
log.info(awaitingLogMessage(context)); log.info(awaitingLogMessage(context));
if (workerShutdownWithRemaining(context.shutdownCompleteLatch().getCount(), context)) { if (workerShutdownWithRemaining(
context.shutdownCompleteLatch().getCount(), context)) {
return false; return false;
} }
} }
} catch (InterruptedException ie) { } catch (InterruptedException ie) {
log.warn("Interrupted while waiting for notification complete, terminating shutdown. {}", log.warn(
"Interrupted while waiting for notification complete, terminating shutdown. {}",
awaitingLogMessage(context)); awaitingLogMessage(context));
return false; return false;
} }
@ -129,12 +134,14 @@ class GracefulShutdownCoordinator {
throw new InterruptedException(); throw new InterruptedException();
} }
log.info(awaitingFinalShutdownMessage(context)); log.info(awaitingFinalShutdownMessage(context));
if (workerShutdownWithRemaining(context.shutdownCompleteLatch().getCount(), context)) { if (workerShutdownWithRemaining(
context.shutdownCompleteLatch().getCount(), context)) {
return false; return false;
} }
} }
} catch (InterruptedException ie) { } catch (InterruptedException ie) {
log.warn("Interrupted while waiting for shutdown completion, terminating shutdown. {}", log.warn(
"Interrupted while waiting for shutdown completion, terminating shutdown. {}",
awaitingFinalShutdownMessage(context)); awaitingFinalShutdownMessage(context));
return false; return false;
} }
@ -152,9 +159,12 @@ class GracefulShutdownCoordinator {
private boolean workerShutdownWithRemaining(long outstanding, GracefulShutdownContext context) { private boolean workerShutdownWithRemaining(long outstanding, GracefulShutdownContext context) {
if (isWorkerShutdownComplete(context)) { if (isWorkerShutdownComplete(context)) {
if (outstanding != 0) { if (outstanding != 0) {
log.info("Shutdown completed, but shutdownCompleteLatch still had outstanding {} with a current" log.info(
+ " value of {}. shutdownComplete: {} -- Consumer Map: {}", outstanding, "Shutdown completed, but shutdownCompleteLatch still had outstanding {} with a current"
context.shutdownCompleteLatch().getCount(), context.scheduler().shutdownComplete(), + " value of {}. shutdownComplete: {} -- Consumer Map: {}",
outstanding,
context.shutdownCompleteLatch().getCount(),
context.scheduler().shutdownComplete(),
context.scheduler().shardInfoShardConsumerMap().size()); context.scheduler().shardInfoShardConsumerMap().size());
return true; return true;
} }

View file

@ -19,12 +19,8 @@ public class NoOpWorkerStateChangeListener implements WorkerStateChangeListener
/** /**
* Empty constructor for NoOp Worker State Change Listener * Empty constructor for NoOp Worker State Change Listener
*/ */
public NoOpWorkerStateChangeListener() { public NoOpWorkerStateChangeListener() {}
}
@Override @Override
public void onWorkerStateChange(WorkerState newState) { public void onWorkerStateChange(WorkerState newState) {}
}
} }

View file

@ -14,6 +14,24 @@
*/ */
package software.amazon.kinesis.coordinator; package software.amazon.kinesis.coordinator;
import java.io.Serializable;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Function;
import java.util.stream.Collectors;
import com.google.common.annotations.VisibleForTesting; import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ComparisonChain; import com.google.common.collect.ComparisonChain;
import lombok.AccessLevel; import lombok.AccessLevel;
@ -46,24 +64,6 @@ import software.amazon.kinesis.metrics.MetricsLevel;
import software.amazon.kinesis.metrics.MetricsScope; import software.amazon.kinesis.metrics.MetricsScope;
import software.amazon.kinesis.metrics.MetricsUtil; import software.amazon.kinesis.metrics.MetricsUtil;
import java.io.Serializable;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Function;
import java.util.stream.Collectors;
import static software.amazon.kinesis.common.HashKeyRangeForLease.fromHashKeyRange; import static software.amazon.kinesis.common.HashKeyRangeForLease.fromHashKeyRange;
/** /**
@ -76,10 +76,13 @@ import static software.amazon.kinesis.common.HashKeyRangeForLease.fromHashKeyRan
@KinesisClientInternalApi @KinesisClientInternalApi
class PeriodicShardSyncManager { class PeriodicShardSyncManager {
private static final long INITIAL_DELAY = 60 * 1000L; private static final long INITIAL_DELAY = 60 * 1000L;
@VisibleForTesting @VisibleForTesting
static final BigInteger MIN_HASH_KEY = BigInteger.ZERO; static final BigInteger MIN_HASH_KEY = BigInteger.ZERO;
@VisibleForTesting @VisibleForTesting
static final BigInteger MAX_HASH_KEY = new BigInteger("2").pow(128).subtract(BigInteger.ONE); static final BigInteger MAX_HASH_KEY = new BigInteger("2").pow(128).subtract(BigInteger.ONE);
static final String PERIODIC_SHARD_SYNC_MANAGER = "PeriodicShardSyncManager"; static final String PERIODIC_SHARD_SYNC_MANAGER = "PeriodicShardSyncManager";
private final Map<StreamIdentifier, HashRangeHoleTracker> hashRangeHoleTrackerMap = new HashMap<>(); private final Map<StreamIdentifier, HashRangeHoleTracker> hashRangeHoleTrackerMap = new HashMap<>();
@ -94,30 +97,48 @@ class PeriodicShardSyncManager {
private final MetricsFactory metricsFactory; private final MetricsFactory metricsFactory;
private final long leasesRecoveryAuditorExecutionFrequencyMillis; private final long leasesRecoveryAuditorExecutionFrequencyMillis;
private final int leasesRecoveryAuditorInconsistencyConfidenceThreshold; private final int leasesRecoveryAuditorInconsistencyConfidenceThreshold;
@Getter(AccessLevel.NONE) @Getter(AccessLevel.NONE)
private final AtomicBoolean leaderSynced; private final AtomicBoolean leaderSynced;
private boolean isRunning; private boolean isRunning;
PeriodicShardSyncManager(String workerId, LeaderDecider leaderDecider, LeaseRefresher leaseRefresher, PeriodicShardSyncManager(
String workerId,
LeaderDecider leaderDecider,
LeaseRefresher leaseRefresher,
Map<StreamIdentifier, StreamConfig> currentStreamConfigMap, Map<StreamIdentifier, StreamConfig> currentStreamConfigMap,
Function<StreamConfig, ShardSyncTaskManager> shardSyncTaskManagerProvider, Function<StreamConfig, ShardSyncTaskManager> shardSyncTaskManagerProvider,
Map<StreamConfig, ShardSyncTaskManager> streamToShardSyncTaskManagerMap, Map<StreamConfig, ShardSyncTaskManager> streamToShardSyncTaskManagerMap,
boolean isMultiStreamingMode, MetricsFactory metricsFactory, boolean isMultiStreamingMode,
MetricsFactory metricsFactory,
long leasesRecoveryAuditorExecutionFrequencyMillis, long leasesRecoveryAuditorExecutionFrequencyMillis,
int leasesRecoveryAuditorInconsistencyConfidenceThreshold, int leasesRecoveryAuditorInconsistencyConfidenceThreshold,
AtomicBoolean leaderSynced){ AtomicBoolean leaderSynced) {
this(workerId, leaderDecider, leaseRefresher, currentStreamConfigMap, shardSyncTaskManagerProvider, this(
workerId,
leaderDecider,
leaseRefresher,
currentStreamConfigMap,
shardSyncTaskManagerProvider,
streamToShardSyncTaskManagerMap, streamToShardSyncTaskManagerMap,
Executors.newSingleThreadScheduledExecutor(), isMultiStreamingMode, metricsFactory, Executors.newSingleThreadScheduledExecutor(),
leasesRecoveryAuditorExecutionFrequencyMillis, leasesRecoveryAuditorInconsistencyConfidenceThreshold, isMultiStreamingMode,
metricsFactory,
leasesRecoveryAuditorExecutionFrequencyMillis,
leasesRecoveryAuditorInconsistencyConfidenceThreshold,
leaderSynced); leaderSynced);
} }
PeriodicShardSyncManager(String workerId, LeaderDecider leaderDecider, LeaseRefresher leaseRefresher, PeriodicShardSyncManager(
String workerId,
LeaderDecider leaderDecider,
LeaseRefresher leaseRefresher,
Map<StreamIdentifier, StreamConfig> currentStreamConfigMap, Map<StreamIdentifier, StreamConfig> currentStreamConfigMap,
Function<StreamConfig, ShardSyncTaskManager> shardSyncTaskManagerProvider, Function<StreamConfig, ShardSyncTaskManager> shardSyncTaskManagerProvider,
Map<StreamConfig, ShardSyncTaskManager> streamToShardSyncTaskManagerMap, Map<StreamConfig, ShardSyncTaskManager> streamToShardSyncTaskManagerMap,
ScheduledExecutorService shardSyncThreadPool, boolean isMultiStreamingMode, ScheduledExecutorService shardSyncThreadPool,
boolean isMultiStreamingMode,
MetricsFactory metricsFactory, MetricsFactory metricsFactory,
long leasesRecoveryAuditorExecutionFrequencyMillis, long leasesRecoveryAuditorExecutionFrequencyMillis,
int leasesRecoveryAuditorInconsistencyConfidenceThreshold, int leasesRecoveryAuditorInconsistencyConfidenceThreshold,
@ -134,7 +155,8 @@ class PeriodicShardSyncManager {
this.isMultiStreamingMode = isMultiStreamingMode; this.isMultiStreamingMode = isMultiStreamingMode;
this.metricsFactory = metricsFactory; this.metricsFactory = metricsFactory;
this.leasesRecoveryAuditorExecutionFrequencyMillis = leasesRecoveryAuditorExecutionFrequencyMillis; this.leasesRecoveryAuditorExecutionFrequencyMillis = leasesRecoveryAuditorExecutionFrequencyMillis;
this.leasesRecoveryAuditorInconsistencyConfidenceThreshold = leasesRecoveryAuditorInconsistencyConfidenceThreshold; this.leasesRecoveryAuditorInconsistencyConfidenceThreshold =
leasesRecoveryAuditorInconsistencyConfidenceThreshold;
this.leaderSynced = leaderSynced; this.leaderSynced = leaderSynced;
} }
@ -147,10 +169,12 @@ class PeriodicShardSyncManager {
log.error("Error during runShardSync.", t); log.error("Error during runShardSync.", t);
} }
}; };
shardSyncThreadPool.scheduleWithFixedDelay(periodicShardSyncer, INITIAL_DELAY, leasesRecoveryAuditorExecutionFrequencyMillis, shardSyncThreadPool.scheduleWithFixedDelay(
periodicShardSyncer,
INITIAL_DELAY,
leasesRecoveryAuditorExecutionFrequencyMillis,
TimeUnit.MILLISECONDS); TimeUnit.MILLISECONDS);
isRunning = true; isRunning = true;
} }
return new TaskResult(null); return new TaskResult(null);
} }
@ -186,8 +210,8 @@ class PeriodicShardSyncManager {
if (leaderDecider.isLeader(workerId) && leaderSynced.get()) { if (leaderDecider.isLeader(workerId) && leaderSynced.get()) {
log.info(String.format("WorkerId %s is leader, running the periodic shard sync task", workerId)); log.info(String.format("WorkerId %s is leader, running the periodic shard sync task", workerId));
final MetricsScope scope = MetricsUtil.createMetricsWithOperation(metricsFactory, final MetricsScope scope =
PERIODIC_SHARD_SYNC_MANAGER); MetricsUtil.createMetricsWithOperation(metricsFactory, PERIODIC_SHARD_SYNC_MANAGER);
int numStreamsWithPartialLeases = 0; int numStreamsWithPartialLeases = 0;
int numStreamsToSync = 0; int numStreamsToSync = 0;
int numSkippedShardSyncTask = 0; int numSkippedShardSyncTask = 0;
@ -207,15 +231,17 @@ class PeriodicShardSyncManager {
log.info("Skipping shard sync task for {} as stream is purged", streamIdentifier); log.info("Skipping shard sync task for {} as stream is purged", streamIdentifier);
continue; continue;
} }
final ShardSyncResponse shardSyncResponse = checkForShardSync(streamIdentifier, final ShardSyncResponse shardSyncResponse =
streamToLeasesMap.get(streamIdentifier)); checkForShardSync(streamIdentifier, streamToLeasesMap.get(streamIdentifier));
numStreamsWithPartialLeases += shardSyncResponse.isHoleDetected() ? 1 : 0; numStreamsWithPartialLeases += shardSyncResponse.isHoleDetected() ? 1 : 0;
numStreamsToSync += shardSyncResponse.shouldDoShardSync ? 1 : 0; numStreamsToSync += shardSyncResponse.shouldDoShardSync ? 1 : 0;
if (shardSyncResponse.shouldDoShardSync()) { if (shardSyncResponse.shouldDoShardSync()) {
log.info("Periodic shard syncer initiating shard sync for {} due to the reason - {} ", log.info(
streamIdentifier, shardSyncResponse.reasonForDecision()); "Periodic shard syncer initiating shard sync for {} due to the reason - {} ",
streamIdentifier,
shardSyncResponse.reasonForDecision());
final StreamConfig streamConfig = currentStreamConfigMap.get(streamIdentifier); final StreamConfig streamConfig = currentStreamConfigMap.get(streamIdentifier);
if (streamConfig == null) { if (streamConfig == null) {
log.info("Skipping shard sync task for {} as stream is purged", streamIdentifier); log.info("Skipping shard sync task for {} as stream is purged", streamIdentifier);
@ -223,15 +249,15 @@ class PeriodicShardSyncManager {
} }
final ShardSyncTaskManager shardSyncTaskManager; final ShardSyncTaskManager shardSyncTaskManager;
if (streamToShardSyncTaskManagerMap.containsKey(streamConfig)) { if (streamToShardSyncTaskManagerMap.containsKey(streamConfig)) {
log.info("shardSyncTaskManager for stream {} already exists", log.info(
streamIdentifier.streamName()); "shardSyncTaskManager for stream {} already exists", streamIdentifier.streamName());
shardSyncTaskManager = streamToShardSyncTaskManagerMap.get(streamConfig); shardSyncTaskManager = streamToShardSyncTaskManagerMap.get(streamConfig);
} } else {
else {
// If streamConfig of a stream has already been added to currentStreamConfigMap but // If streamConfig of a stream has already been added to currentStreamConfigMap but
// Scheduler failed to create shardSyncTaskManager for it, then Scheduler will not try // Scheduler failed to create shardSyncTaskManager for it, then Scheduler will not try
// to create one later. So enable PeriodicShardSyncManager to do it for such cases // to create one later. So enable PeriodicShardSyncManager to do it for such cases
log.info("Failed to get shardSyncTaskManager so creating one for stream {}.", log.info(
"Failed to get shardSyncTaskManager so creating one for stream {}.",
streamIdentifier.streamName()); streamIdentifier.streamName());
shardSyncTaskManager = streamToShardSyncTaskManagerMap.computeIfAbsent( shardSyncTaskManager = streamToShardSyncTaskManagerMap.computeIfAbsent(
streamConfig, s -> shardSyncTaskManagerProvider.apply(s)); streamConfig, s -> shardSyncTaskManagerProvider.apply(s));
@ -239,15 +265,24 @@ class PeriodicShardSyncManager {
if (!shardSyncTaskManager.submitShardSyncTask()) { if (!shardSyncTaskManager.submitShardSyncTask()) {
log.warn( log.warn(
"Failed to submit shard sync task for stream {}. This could be due to the previous pending shard sync task.", "Failed to submit shard sync task for stream {}. This could be due to the previous pending shard sync task.",
shardSyncTaskManager.shardDetector().streamIdentifier().streamName()); shardSyncTaskManager
.shardDetector()
.streamIdentifier()
.streamName());
numSkippedShardSyncTask += 1; numSkippedShardSyncTask += 1;
} else { } else {
log.info("Submitted shard sync task for stream {} because of reason {}", log.info(
shardSyncTaskManager.shardDetector().streamIdentifier().streamName(), "Submitted shard sync task for stream {} because of reason {}",
shardSyncTaskManager
.shardDetector()
.streamIdentifier()
.streamName(),
shardSyncResponse.reasonForDecision()); shardSyncResponse.reasonForDecision());
} }
} else { } else {
log.info("Skipping shard sync for {} due to the reason - {}", streamIdentifier, log.info(
"Skipping shard sync for {} due to the reason - {}",
streamIdentifier,
shardSyncResponse.reasonForDecision()); shardSyncResponse.reasonForDecision());
} }
} }
@ -255,9 +290,14 @@ class PeriodicShardSyncManager {
} catch (Exception e) { } catch (Exception e) {
log.error("Caught exception while running periodic shard syncer.", e); log.error("Caught exception while running periodic shard syncer.", e);
} finally { } finally {
scope.addData("NumStreamsWithPartialLeases", numStreamsWithPartialLeases, StandardUnit.COUNT, MetricsLevel.SUMMARY); scope.addData(
"NumStreamsWithPartialLeases",
numStreamsWithPartialLeases,
StandardUnit.COUNT,
MetricsLevel.SUMMARY);
scope.addData("NumStreamsToSync", numStreamsToSync, StandardUnit.COUNT, MetricsLevel.SUMMARY); scope.addData("NumStreamsToSync", numStreamsToSync, StandardUnit.COUNT, MetricsLevel.SUMMARY);
scope.addData("NumSkippedShardSyncTask", numSkippedShardSyncTask, StandardUnit.COUNT, MetricsLevel.SUMMARY); scope.addData(
"NumSkippedShardSyncTask", numSkippedShardSyncTask, StandardUnit.COUNT, MetricsLevel.SUMMARY);
MetricsUtil.addSuccessAndLatency(scope, isRunSuccess, runStartMillis, MetricsLevel.SUMMARY); MetricsUtil.addSuccessAndLatency(scope, isRunSuccess, runStartMillis, MetricsLevel.SUMMARY);
scope.end(); scope.end();
} }
@ -284,17 +324,18 @@ class PeriodicShardSyncManager {
} else { } else {
final Map<StreamIdentifier, List<Lease>> streamToLeasesMap = new HashMap<>(); final Map<StreamIdentifier, List<Lease>> streamToLeasesMap = new HashMap<>();
for (Lease lease : leases) { for (Lease lease : leases) {
StreamIdentifier streamIdentifier = StreamIdentifier StreamIdentifier streamIdentifier =
.multiStreamInstance(((MultiStreamLease) lease).streamIdentifier()); StreamIdentifier.multiStreamInstance(((MultiStreamLease) lease).streamIdentifier());
if (streamIdentifiersToFilter.contains(streamIdentifier)) { if (streamIdentifiersToFilter.contains(streamIdentifier)) {
streamToLeasesMap.computeIfAbsent(streamIdentifier, s -> new ArrayList<>()).add(lease); streamToLeasesMap
.computeIfAbsent(streamIdentifier, s -> new ArrayList<>())
.add(lease);
} }
} }
return streamToLeasesMap; return streamToLeasesMap;
} }
} }
/** /**
* Given a list of leases for a stream, determine if a shard sync is necessary. * Given a list of leases for a stream, determine if a shard sync is necessary.
* @param streamIdentifier * @param streamIdentifier
@ -315,11 +356,13 @@ class PeriodicShardSyncManager {
// If hole is determined with high confidence return true; return false otherwise // If hole is determined with high confidence return true; return false otherwise
// We are using the high confidence factor to avoid shard sync on any holes during resharding and // We are using the high confidence factor to avoid shard sync on any holes during resharding and
// lease cleanups or any intermittent issues. // lease cleanups or any intermittent issues.
final HashRangeHoleTracker hashRangeHoleTracker = hashRangeHoleTrackerMap final HashRangeHoleTracker hashRangeHoleTracker =
.computeIfAbsent(streamIdentifier, s -> new HashRangeHoleTracker()); hashRangeHoleTrackerMap.computeIfAbsent(streamIdentifier, s -> new HashRangeHoleTracker());
final boolean hasHoleWithHighConfidence = hashRangeHoleTracker final boolean hasHoleWithHighConfidence =
.hasHighConfidenceOfHoleWith(hashRangeHoleOpt.get()); hashRangeHoleTracker.hasHighConfidenceOfHoleWith(hashRangeHoleOpt.get());
return new ShardSyncResponse(hasHoleWithHighConfidence, true, return new ShardSyncResponse(
hasHoleWithHighConfidence,
true,
"Detected same hole for " + hashRangeHoleTracker.getNumConsecutiveHoles() "Detected same hole for " + hashRangeHoleTracker.getNumConsecutiveHoles()
+ " times. Shard sync will be initiated when threshold reaches " + " times. Shard sync will be initiated when threshold reaches "
+ leasesRecoveryAuditorInconsistencyConfidenceThreshold); + leasesRecoveryAuditorInconsistencyConfidenceThreshold);
@ -355,7 +398,9 @@ class PeriodicShardSyncManager {
Optional<HashRangeHole> hasHoleInLeases(StreamIdentifier streamIdentifier, List<Lease> leases) { Optional<HashRangeHole> hasHoleInLeases(StreamIdentifier streamIdentifier, List<Lease> leases) {
// Filter the leases with any checkpoint other than shard end. // Filter the leases with any checkpoint other than shard end.
List<Lease> activeLeases = leases.stream() List<Lease> activeLeases = leases.stream()
.filter(lease -> lease.checkpoint() != null && !lease.checkpoint().isShardEnd()).collect(Collectors.toList()); .filter(lease ->
lease.checkpoint() != null && !lease.checkpoint().isShardEnd())
.collect(Collectors.toList());
List<Lease> activeLeasesWithHashRanges = fillWithHashRangesIfRequired(streamIdentifier, activeLeases); List<Lease> activeLeasesWithHashRanges = fillWithHashRangesIfRequired(streamIdentifier, activeLeases);
return checkForHoleInHashKeyRanges(streamIdentifier, activeLeasesWithHashRanges); return checkForHoleInHashKeyRanges(streamIdentifier, activeLeasesWithHashRanges);
} }
@ -364,43 +409,49 @@ class PeriodicShardSyncManager {
// by learning from kinesis shards. // by learning from kinesis shards.
private List<Lease> fillWithHashRangesIfRequired(StreamIdentifier streamIdentifier, List<Lease> activeLeases) { private List<Lease> fillWithHashRangesIfRequired(StreamIdentifier streamIdentifier, List<Lease> activeLeases) {
List<Lease> activeLeasesWithNoHashRanges = activeLeases.stream() List<Lease> activeLeasesWithNoHashRanges = activeLeases.stream()
.filter(lease -> lease.hashKeyRangeForLease() == null).collect(Collectors.toList()); .filter(lease -> lease.hashKeyRangeForLease() == null)
.collect(Collectors.toList());
Optional<Lease> minLeaseOpt = activeLeasesWithNoHashRanges.stream().min(Comparator.comparing(Lease::leaseKey)); Optional<Lease> minLeaseOpt = activeLeasesWithNoHashRanges.stream().min(Comparator.comparing(Lease::leaseKey));
if (minLeaseOpt.isPresent()) { if (minLeaseOpt.isPresent()) {
// TODO : use minLease for new ListShards with startingShardId // TODO : use minLease for new ListShards with startingShardId
final Lease minLease = minLeaseOpt.get(); final Lease minLease = minLeaseOpt.get();
final ShardDetector shardDetector = shardSyncTaskManagerProvider final ShardDetector shardDetector = shardSyncTaskManagerProvider
.apply(currentStreamConfigMap.get(streamIdentifier)).shardDetector(); .apply(currentStreamConfigMap.get(streamIdentifier))
final Map<String, Shard> kinesisShards = shardDetector.listShards().stream() .shardDetector();
.collect(Collectors.toMap(Shard::shardId, shard -> shard)); final Map<String, Shard> kinesisShards =
return activeLeases.stream().map(lease -> { shardDetector.listShards().stream().collect(Collectors.toMap(Shard::shardId, shard -> shard));
if (lease.hashKeyRangeForLease() == null) { return activeLeases.stream()
final String shardId = lease instanceof MultiStreamLease ? .map(lease -> {
((MultiStreamLease) lease).shardId() : if (lease.hashKeyRangeForLease() == null) {
lease.leaseKey(); final String shardId = lease instanceof MultiStreamLease
final Shard shard = kinesisShards.get(shardId); ? ((MultiStreamLease) lease).shardId()
if (shard == null) { : lease.leaseKey();
final Shard shard = kinesisShards.get(shardId);
if (shard == null) {
return lease;
}
lease.hashKeyRange(fromHashKeyRange(shard.hashKeyRange()));
try {
leaseRefresher.updateLeaseWithMetaInfo(lease, UpdateField.HASH_KEY_RANGE);
} catch (Exception e) {
log.warn(
"Unable to update hash range key information for lease {} of stream {}. This may result in explicit lease sync.",
lease.leaseKey(),
streamIdentifier);
}
}
return lease; return lease;
} })
lease.hashKeyRange(fromHashKeyRange(shard.hashKeyRange())); .filter(lease -> lease.hashKeyRangeForLease() != null)
try { .collect(Collectors.toList());
leaseRefresher.updateLeaseWithMetaInfo(lease, UpdateField.HASH_KEY_RANGE);
} catch (Exception e) {
log.warn(
"Unable to update hash range key information for lease {} of stream {}. This may result in explicit lease sync.",
lease.leaseKey(), streamIdentifier);
}
}
return lease;
}).filter(lease -> lease.hashKeyRangeForLease() != null).collect(Collectors.toList());
} else { } else {
return activeLeases; return activeLeases;
} }
} }
@VisibleForTesting @VisibleForTesting
static Optional<HashRangeHole> checkForHoleInHashKeyRanges(StreamIdentifier streamIdentifier, static Optional<HashRangeHole> checkForHoleInHashKeyRanges(
List<Lease> leasesWithHashKeyRanges) { StreamIdentifier streamIdentifier, List<Lease> leasesWithHashKeyRanges) {
// Sort the hash ranges by starting hash key. // Sort the hash ranges by starting hash key.
List<Lease> sortedLeasesWithHashKeyRanges = sortLeasesByHashRange(leasesWithHashKeyRanges); List<Lease> sortedLeasesWithHashKeyRanges = sortLeasesByHashRange(leasesWithHashKeyRanges);
if (sortedLeasesWithHashKeyRanges.isEmpty()) { if (sortedLeasesWithHashKeyRanges.isEmpty()) {
@ -408,34 +459,54 @@ class PeriodicShardSyncManager {
return Optional.of(new HashRangeHole()); return Optional.of(new HashRangeHole());
} }
// Validate for hashranges bounds. // Validate for hashranges bounds.
if (!sortedLeasesWithHashKeyRanges.get(0).hashKeyRangeForLease().startingHashKey().equals(MIN_HASH_KEY) || !sortedLeasesWithHashKeyRanges if (!sortedLeasesWithHashKeyRanges
.get(sortedLeasesWithHashKeyRanges.size() - 1).hashKeyRangeForLease().endingHashKey().equals(MAX_HASH_KEY)) { .get(0)
log.error("Incomplete hash range found for stream {} between {} and {}.", streamIdentifier, .hashKeyRangeForLease()
.startingHashKey()
.equals(MIN_HASH_KEY)
|| !sortedLeasesWithHashKeyRanges
.get(sortedLeasesWithHashKeyRanges.size() - 1)
.hashKeyRangeForLease()
.endingHashKey()
.equals(MAX_HASH_KEY)) {
log.error(
"Incomplete hash range found for stream {} between {} and {}.",
streamIdentifier,
sortedLeasesWithHashKeyRanges.get(0), sortedLeasesWithHashKeyRanges.get(0),
sortedLeasesWithHashKeyRanges.get(sortedLeasesWithHashKeyRanges.size() - 1)); sortedLeasesWithHashKeyRanges.get(sortedLeasesWithHashKeyRanges.size() - 1));
return Optional.of(new HashRangeHole(sortedLeasesWithHashKeyRanges.get(0).hashKeyRangeForLease(), return Optional.of(new HashRangeHole(
sortedLeasesWithHashKeyRanges.get(sortedLeasesWithHashKeyRanges.size() - 1).hashKeyRangeForLease())); sortedLeasesWithHashKeyRanges.get(0).hashKeyRangeForLease(),
sortedLeasesWithHashKeyRanges
.get(sortedLeasesWithHashKeyRanges.size() - 1)
.hashKeyRangeForLease()));
} }
// Check for any holes in the sorted hashrange intervals. // Check for any holes in the sorted hashrange intervals.
if (sortedLeasesWithHashKeyRanges.size() > 1) { if (sortedLeasesWithHashKeyRanges.size() > 1) {
Lease leftMostLeaseToReportInCaseOfHole = sortedLeasesWithHashKeyRanges.get(0); Lease leftMostLeaseToReportInCaseOfHole = sortedLeasesWithHashKeyRanges.get(0);
HashKeyRangeForLease leftLeaseHashRange = leftMostLeaseToReportInCaseOfHole.hashKeyRangeForLease(); HashKeyRangeForLease leftLeaseHashRange = leftMostLeaseToReportInCaseOfHole.hashKeyRangeForLease();
for (int i = 1; i < sortedLeasesWithHashKeyRanges.size(); i++) { for (int i = 1; i < sortedLeasesWithHashKeyRanges.size(); i++) {
final HashKeyRangeForLease rightLeaseHashRange = sortedLeasesWithHashKeyRanges.get(i).hashKeyRangeForLease(); final HashKeyRangeForLease rightLeaseHashRange =
final BigInteger rangeDiff = rightLeaseHashRange.startingHashKey().subtract(leftLeaseHashRange.endingHashKey()); sortedLeasesWithHashKeyRanges.get(i).hashKeyRangeForLease();
final BigInteger rangeDiff =
rightLeaseHashRange.startingHashKey().subtract(leftLeaseHashRange.endingHashKey());
// Case of overlapping leases when the rangediff is 0 or negative. // Case of overlapping leases when the rangediff is 0 or negative.
// signum() will be -1 for negative and 0 if value is 0. // signum() will be -1 for negative and 0 if value is 0.
// Merge the range for further tracking. // Merge the range for further tracking.
if (rangeDiff.signum() <= 0) { if (rangeDiff.signum() <= 0) {
leftLeaseHashRange = new HashKeyRangeForLease(leftLeaseHashRange.startingHashKey(), leftLeaseHashRange = new HashKeyRangeForLease(
leftLeaseHashRange.startingHashKey(),
leftLeaseHashRange.endingHashKey().max(rightLeaseHashRange.endingHashKey())); leftLeaseHashRange.endingHashKey().max(rightLeaseHashRange.endingHashKey()));
} else { } else {
// Case of non overlapping leases when rangediff is positive. signum() will be 1 for positive. // Case of non overlapping leases when rangediff is positive. signum() will be 1 for positive.
// If rangeDiff is 1, then it is a case of continuous hashrange. If not, it is a hole. // If rangeDiff is 1, then it is a case of continuous hashrange. If not, it is a hole.
if (!rangeDiff.equals(BigInteger.ONE)) { if (!rangeDiff.equals(BigInteger.ONE)) {
log.error("Incomplete hash range found for {} between {} and {}.", streamIdentifier, log.error(
leftMostLeaseToReportInCaseOfHole, sortedLeasesWithHashKeyRanges.get(i)); "Incomplete hash range found for {} between {} and {}.",
return Optional.of(new HashRangeHole(leftMostLeaseToReportInCaseOfHole.hashKeyRangeForLease(), streamIdentifier,
leftMostLeaseToReportInCaseOfHole,
sortedLeasesWithHashKeyRanges.get(i));
return Optional.of(new HashRangeHole(
leftMostLeaseToReportInCaseOfHole.hashKeyRangeForLease(),
sortedLeasesWithHashKeyRanges.get(i).hashKeyRangeForLease())); sortedLeasesWithHashKeyRanges.get(i).hashKeyRangeForLease()));
} }
leftMostLeaseToReportInCaseOfHole = sortedLeasesWithHashKeyRanges.get(i); leftMostLeaseToReportInCaseOfHole = sortedLeasesWithHashKeyRanges.get(i);
@ -461,7 +532,9 @@ class PeriodicShardSyncManager {
hashRangeAtStartOfPossibleHole = hashRangeAtEndOfPossibleHole = null; hashRangeAtStartOfPossibleHole = hashRangeAtEndOfPossibleHole = null;
} }
HashRangeHole(HashKeyRangeForLease hashRangeAtStartOfPossibleHole, HashKeyRangeForLease hashRangeAtEndOfPossibleHole) { HashRangeHole(
HashKeyRangeForLease hashRangeAtStartOfPossibleHole,
HashKeyRangeForLease hashRangeAtEndOfPossibleHole) {
this.hashRangeAtStartOfPossibleHole = hashRangeAtStartOfPossibleHole; this.hashRangeAtStartOfPossibleHole = hashRangeAtStartOfPossibleHole;
this.hashRangeAtEndOfPossibleHole = hashRangeAtEndOfPossibleHole; this.hashRangeAtEndOfPossibleHole = hashRangeAtEndOfPossibleHole;
} }
@ -472,6 +545,7 @@ class PeriodicShardSyncManager {
private class HashRangeHoleTracker { private class HashRangeHoleTracker {
private HashRangeHole hashRangeHole; private HashRangeHole hashRangeHole;
@Getter @Getter
private Integer numConsecutiveHoles; private Integer numConsecutiveHoles;
@ -500,8 +574,12 @@ class PeriodicShardSyncManager {
Validate.notNull(lease.hashKeyRangeForLease()); Validate.notNull(lease.hashKeyRangeForLease());
Validate.notNull(otherLease.hashKeyRangeForLease()); Validate.notNull(otherLease.hashKeyRangeForLease());
return ComparisonChain.start() return ComparisonChain.start()
.compare(lease.hashKeyRangeForLease().startingHashKey(), otherLease.hashKeyRangeForLease().startingHashKey()) .compare(
.compare(lease.hashKeyRangeForLease().endingHashKey(), otherLease.hashKeyRangeForLease().endingHashKey()) lease.hashKeyRangeForLease().startingHashKey(),
otherLease.hashKeyRangeForLease().startingHashKey())
.compare(
lease.hashKeyRangeForLease().endingHashKey(),
otherLease.hashKeyRangeForLease().endingHashKey())
.result(); .result();
} }
} }

View file

@ -25,8 +25,8 @@ import software.amazon.kinesis.annotations.KinesisClientInternalApi;
@Slf4j @Slf4j
@KinesisClientInternalApi @KinesisClientInternalApi
class RejectedTaskEvent implements DiagnosticEvent { class RejectedTaskEvent implements DiagnosticEvent {
private static final String MESSAGE = "Review your thread configuration to prevent task rejections. " + private static final String MESSAGE = "Review your thread configuration to prevent task rejections. "
"Task rejections will slow down your application and some shards may stop processing. "; + "Task rejections will slow down your application and some shards may stop processing. ";
private ExecutorStateEvent executorStateEvent; private ExecutorStateEvent executorStateEvent;
private Throwable throwable; private Throwable throwable;

View file

@ -15,10 +15,6 @@
package software.amazon.kinesis.coordinator; package software.amazon.kinesis.coordinator;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Stopwatch;
import io.reactivex.rxjava3.plugins.RxJavaPlugins;
import java.time.Duration; import java.time.Duration;
import java.time.Instant; import java.time.Instant;
import java.util.ArrayList; import java.util.ArrayList;
@ -45,6 +41,10 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Consumer; import java.util.function.Consumer;
import java.util.function.Function; import java.util.function.Function;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Stopwatch;
import io.reactivex.rxjava3.plugins.RxJavaPlugins;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.Getter; import lombok.Getter;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
@ -188,6 +188,7 @@ public class Scheduler implements Runnable {
private final Stopwatch streamSyncWatch = Stopwatch.createUnstarted(); private final Stopwatch streamSyncWatch = Stopwatch.createUnstarted();
private boolean leasesSyncedOnAppInit = false; private boolean leasesSyncedOnAppInit = false;
@Getter(AccessLevel.NONE) @Getter(AccessLevel.NONE)
private final AtomicBoolean leaderSynced = new AtomicBoolean(false); private final AtomicBoolean leaderSynced = new AtomicBoolean(false);
@ -206,15 +207,23 @@ public class Scheduler implements Runnable {
@VisibleForTesting @VisibleForTesting
protected boolean gracefuleShutdownStarted = false; protected boolean gracefuleShutdownStarted = false;
public Scheduler(@NonNull final CheckpointConfig checkpointConfig, public Scheduler(
@NonNull final CoordinatorConfig coordinatorConfig, @NonNull final CheckpointConfig checkpointConfig,
@NonNull final LeaseManagementConfig leaseManagementConfig, @NonNull final CoordinatorConfig coordinatorConfig,
@NonNull final LifecycleConfig lifecycleConfig, @NonNull final LeaseManagementConfig leaseManagementConfig,
@NonNull final MetricsConfig metricsConfig, @NonNull final LifecycleConfig lifecycleConfig,
@NonNull final ProcessorConfig processorConfig, @NonNull final MetricsConfig metricsConfig,
@NonNull final RetrievalConfig retrievalConfig) { @NonNull final ProcessorConfig processorConfig,
this(checkpointConfig, coordinatorConfig, leaseManagementConfig, lifecycleConfig, metricsConfig, @NonNull final RetrievalConfig retrievalConfig) {
processorConfig, retrievalConfig, new DiagnosticEventFactory()); this(
checkpointConfig,
coordinatorConfig,
leaseManagementConfig,
lifecycleConfig,
metricsConfig,
processorConfig,
retrievalConfig,
new DiagnosticEventFactory());
} }
/** /**
@ -222,14 +231,15 @@ public class Scheduler implements Runnable {
* is desired for testing. This constructor is only used for testing to provide a mock DiagnosticEventFactory. * is desired for testing. This constructor is only used for testing to provide a mock DiagnosticEventFactory.
*/ */
@VisibleForTesting @VisibleForTesting
protected Scheduler(@NonNull final CheckpointConfig checkpointConfig, protected Scheduler(
@NonNull final CoordinatorConfig coordinatorConfig, @NonNull final CheckpointConfig checkpointConfig,
@NonNull final LeaseManagementConfig leaseManagementConfig, @NonNull final CoordinatorConfig coordinatorConfig,
@NonNull final LifecycleConfig lifecycleConfig, @NonNull final LeaseManagementConfig leaseManagementConfig,
@NonNull final MetricsConfig metricsConfig, @NonNull final LifecycleConfig lifecycleConfig,
@NonNull final ProcessorConfig processorConfig, @NonNull final MetricsConfig metricsConfig,
@NonNull final RetrievalConfig retrievalConfig, @NonNull final ProcessorConfig processorConfig,
@NonNull final DiagnosticEventFactory diagnosticEventFactory) { @NonNull final RetrievalConfig retrievalConfig,
@NonNull final DiagnosticEventFactory diagnosticEventFactory) {
this.checkpointConfig = checkpointConfig; this.checkpointConfig = checkpointConfig;
this.coordinatorConfig = coordinatorConfig; this.coordinatorConfig = coordinatorConfig;
this.leaseManagementConfig = leaseManagementConfig; this.leaseManagementConfig = leaseManagementConfig;
@ -242,16 +252,14 @@ public class Scheduler implements Runnable {
this.streamTracker = retrievalConfig.streamTracker(); this.streamTracker = retrievalConfig.streamTracker();
this.isMultiStreamMode = streamTracker.isMultiStream(); this.isMultiStreamMode = streamTracker.isMultiStream();
this.formerStreamsLeasesDeletionStrategy = streamTracker.formerStreamsLeasesDeletionStrategy(); this.formerStreamsLeasesDeletionStrategy = streamTracker.formerStreamsLeasesDeletionStrategy();
streamTracker.streamConfigList().forEach( streamTracker.streamConfigList().forEach(sc -> currentStreamConfigMap.put(sc.streamIdentifier(), sc));
sc -> currentStreamConfigMap.put(sc.streamIdentifier(), sc));
log.info("Initial state: {}", currentStreamConfigMap.values()); log.info("Initial state: {}", currentStreamConfigMap.values());
this.maxInitializationAttempts = this.coordinatorConfig.maxInitializationAttempts(); this.maxInitializationAttempts = this.coordinatorConfig.maxInitializationAttempts();
this.metricsFactory = this.metricsConfig.metricsFactory(); this.metricsFactory = this.metricsConfig.metricsFactory();
// Determine leaseSerializer based on availability of MultiStreamTracker. // Determine leaseSerializer based on availability of MultiStreamTracker.
final LeaseSerializer leaseSerializer = isMultiStreamMode ? final LeaseSerializer leaseSerializer =
new DynamoDBMultiStreamLeaseSerializer() : isMultiStreamMode ? new DynamoDBMultiStreamLeaseSerializer() : new DynamoDBLeaseSerializer();
new DynamoDBLeaseSerializer();
this.leaseCoordinator = this.leaseManagementConfig this.leaseCoordinator = this.leaseManagementConfig
.leaseManagementFactory(leaseSerializer, isMultiStreamMode) .leaseManagementFactory(leaseSerializer, isMultiStreamMode)
.createLeaseCoordinator(this.metricsFactory); .createLeaseCoordinator(this.metricsFactory);
@ -260,8 +268,9 @@ public class Scheduler implements Runnable {
// //
// TODO: Figure out what to do with lease manage <=> checkpoint relationship // TODO: Figure out what to do with lease manage <=> checkpoint relationship
// //
this.checkpoint = this.checkpointConfig.checkpointFactory().createCheckpointer(this.leaseCoordinator, this.checkpoint = this.checkpointConfig
this.leaseRefresher); .checkpointFactory()
.createCheckpointer(this.leaseCoordinator, this.leaseRefresher);
// //
// TODO: Move this configuration to lifecycle // TODO: Move this configuration to lifecycle
@ -282,38 +291,46 @@ public class Scheduler implements Runnable {
if (coordinatorConfig.gracefulShutdownCoordinator() != null) { if (coordinatorConfig.gracefulShutdownCoordinator() != null) {
this.gracefulShutdownCoordinator = coordinatorConfig.gracefulShutdownCoordinator(); this.gracefulShutdownCoordinator = coordinatorConfig.gracefulShutdownCoordinator();
} else { } else {
this.gracefulShutdownCoordinator = this.coordinatorConfig.coordinatorFactory() this.gracefulShutdownCoordinator =
.createGracefulShutdownCoordinator(); this.coordinatorConfig.coordinatorFactory().createGracefulShutdownCoordinator();
} }
if (coordinatorConfig.workerStateChangeListener() != null) { if (coordinatorConfig.workerStateChangeListener() != null) {
this.workerStateChangeListener = coordinatorConfig.workerStateChangeListener(); this.workerStateChangeListener = coordinatorConfig.workerStateChangeListener();
} else { } else {
this.workerStateChangeListener = this.coordinatorConfig.coordinatorFactory() this.workerStateChangeListener =
.createWorkerStateChangeListener(); this.coordinatorConfig.coordinatorFactory().createWorkerStateChangeListener();
} }
this.leaderDecider = new DeterministicShuffleShardSyncLeaderDecider(leaseRefresher, this.leaderDecider = new DeterministicShuffleShardSyncLeaderDecider(
Executors.newSingleThreadScheduledExecutor(), leaseRefresher, Executors.newSingleThreadScheduledExecutor(), PERIODIC_SHARD_SYNC_MAX_WORKERS_DEFAULT);
PERIODIC_SHARD_SYNC_MAX_WORKERS_DEFAULT);
this.failoverTimeMillis = this.leaseManagementConfig.failoverTimeMillis(); this.failoverTimeMillis = this.leaseManagementConfig.failoverTimeMillis();
this.taskBackoffTimeMillis = this.lifecycleConfig.taskBackoffTimeMillis(); this.taskBackoffTimeMillis = this.lifecycleConfig.taskBackoffTimeMillis();
this.listShardsBackoffTimeMillis = this.retrievalConfig.listShardsBackoffTimeInMillis(); this.listShardsBackoffTimeMillis = this.retrievalConfig.listShardsBackoffTimeInMillis();
this.maxListShardsRetryAttempts = this.retrievalConfig.maxListShardsRetryAttempts(); this.maxListShardsRetryAttempts = this.retrievalConfig.maxListShardsRetryAttempts();
this.shardDetectorProvider = streamConfig -> createOrGetShardSyncTaskManager(streamConfig).shardDetector(); this.shardDetectorProvider =
streamConfig -> createOrGetShardSyncTaskManager(streamConfig).shardDetector();
this.ignoreUnexpetedChildShards = this.leaseManagementConfig.ignoreUnexpectedChildShards(); this.ignoreUnexpetedChildShards = this.leaseManagementConfig.ignoreUnexpectedChildShards();
this.aggregatorUtil = this.lifecycleConfig.aggregatorUtil(); this.aggregatorUtil = this.lifecycleConfig.aggregatorUtil();
this.hierarchicalShardSyncerProvider = streamConfig -> createOrGetShardSyncTaskManager(streamConfig).hierarchicalShardSyncer(); this.hierarchicalShardSyncerProvider =
this.schedulerInitializationBackoffTimeMillis = this.coordinatorConfig.schedulerInitializationBackoffTimeMillis(); streamConfig -> createOrGetShardSyncTaskManager(streamConfig).hierarchicalShardSyncer();
this.schedulerInitializationBackoffTimeMillis =
this.coordinatorConfig.schedulerInitializationBackoffTimeMillis();
this.leaderElectedPeriodicShardSyncManager = new PeriodicShardSyncManager( this.leaderElectedPeriodicShardSyncManager = new PeriodicShardSyncManager(
leaseManagementConfig.workerIdentifier(), leaderDecider, leaseRefresher, currentStreamConfigMap, leaseManagementConfig.workerIdentifier(),
shardSyncTaskManagerProvider, streamToShardSyncTaskManagerMap, isMultiStreamMode, metricsFactory, leaderDecider,
leaseRefresher,
currentStreamConfigMap,
shardSyncTaskManagerProvider,
streamToShardSyncTaskManagerMap,
isMultiStreamMode,
metricsFactory,
leaseManagementConfig.leasesRecoveryAuditorExecutionFrequencyMillis(), leaseManagementConfig.leasesRecoveryAuditorExecutionFrequencyMillis(),
leaseManagementConfig.leasesRecoveryAuditorInconsistencyConfidenceThreshold(), leaseManagementConfig.leasesRecoveryAuditorInconsistencyConfidenceThreshold(),
leaderSynced); leaderSynced);
this.leaseCleanupManager = this.leaseManagementConfig.leaseManagementFactory(leaseSerializer, isMultiStreamMode) this.leaseCleanupManager = this.leaseManagementConfig
.leaseManagementFactory(leaseSerializer, isMultiStreamMode)
.createLeaseCleanupManager(metricsFactory); .createLeaseCleanupManager(metricsFactory);
this.schemaRegistryDecoder = this.schemaRegistryDecoder = this.retrievalConfig.glueSchemaRegistryDeserializer() == null
this.retrievalConfig.glueSchemaRegistryDeserializer() == null ? ? null
null
: new SchemaRegistryDecoder(this.retrievalConfig.glueSchemaRegistryDeserializer()); : new SchemaRegistryDecoder(this.retrievalConfig.glueSchemaRegistryDeserializer());
} }
@ -357,9 +374,10 @@ public class Scheduler implements Runnable {
if (!skipShardSyncAtWorkerInitializationIfLeasesExist || leaseRefresher.isLeaseTableEmpty()) { if (!skipShardSyncAtWorkerInitializationIfLeasesExist || leaseRefresher.isLeaseTableEmpty()) {
if (shouldInitiateLeaseSync()) { if (shouldInitiateLeaseSync()) {
log.info("Worker {} is initiating the lease sync.", leaseManagementConfig.workerIdentifier()); log.info(
"Worker {} is initiating the lease sync.",
leaseManagementConfig.workerIdentifier());
leaderElectedPeriodicShardSyncManager.syncShardsOnce(); leaderElectedPeriodicShardSyncManager.syncShardsOnce();
} }
} else { } else {
log.info("Skipping shard sync per configuration setting (and lease table is not empty)"); log.info("Skipping shard sync per configuration setting (and lease table is not empty)");
@ -402,13 +420,15 @@ public class Scheduler implements Runnable {
} }
@VisibleForTesting @VisibleForTesting
boolean shouldInitiateLeaseSync() throws InterruptedException, boolean shouldInitiateLeaseSync()
DependencyException, ProvisionedThroughputException, InvalidStateException { throws InterruptedException, DependencyException, ProvisionedThroughputException, InvalidStateException {
long waitTime = ThreadLocalRandom.current().nextLong(MIN_WAIT_TIME_FOR_LEASE_TABLE_CHECK_MILLIS, MAX_WAIT_TIME_FOR_LEASE_TABLE_CHECK_MILLIS); long waitTime = ThreadLocalRandom.current()
.nextLong(MIN_WAIT_TIME_FOR_LEASE_TABLE_CHECK_MILLIS, MAX_WAIT_TIME_FOR_LEASE_TABLE_CHECK_MILLIS);
long waitUntil = System.currentTimeMillis() + waitTime; long waitUntil = System.currentTimeMillis() + waitTime;
boolean shouldInitiateLeaseSync = true; boolean shouldInitiateLeaseSync = true;
while (System.currentTimeMillis() < waitUntil && (shouldInitiateLeaseSync = leaseRefresher.isLeaseTableEmpty())) { while (System.currentTimeMillis() < waitUntil
&& (shouldInitiateLeaseSync = leaseRefresher.isLeaseTableEmpty())) {
// check every 3 seconds if lease table is still empty, // check every 3 seconds if lease table is still empty,
// to minimize contention between all workers bootstrapping at the same time // to minimize contention between all workers bootstrapping at the same time
log.info("Lease table is still empty. Checking again in {} ms", LEASE_TABLE_CHECK_FREQUENCY_MILLIS); log.info("Lease table is still empty. Checking again in {} ms", LEASE_TABLE_CHECK_FREQUENCY_MILLIS);
@ -422,8 +442,8 @@ public class Scheduler implements Runnable {
try { try {
Set<ShardInfo> assignedShards = new HashSet<>(); Set<ShardInfo> assignedShards = new HashSet<>();
for (ShardInfo shardInfo : getShardInfoForAssignments()) { for (ShardInfo shardInfo : getShardInfoForAssignments()) {
ShardConsumer shardConsumer = createOrGetShardConsumer(shardInfo, ShardConsumer shardConsumer = createOrGetShardConsumer(
processorConfig.shardRecordProcessorFactory(), leaseCleanupManager); shardInfo, processorConfig.shardRecordProcessorFactory(), leaseCleanupManager);
shardConsumer.executeLifecycle(); shardConsumer.executeLifecycle();
assignedShards.add(shardInfo); assignedShards.add(shardInfo);
@ -444,8 +464,10 @@ public class Scheduler implements Runnable {
slog.info("Sleeping ..."); slog.info("Sleeping ...");
Thread.sleep(shardConsumerDispatchPollIntervalMillis); Thread.sleep(shardConsumerDispatchPollIntervalMillis);
} catch (Exception e) { } catch (Exception e) {
log.error("Worker.run caught exception, sleeping for {} milli seconds!", log.error(
shardConsumerDispatchPollIntervalMillis, e); "Worker.run caught exception, sleeping for {} milli seconds!",
shardConsumerDispatchPollIntervalMillis,
e);
try { try {
Thread.sleep(shardConsumerDispatchPollIntervalMillis); Thread.sleep(shardConsumerDispatchPollIntervalMillis);
} catch (InterruptedException ex) { } catch (InterruptedException ex) {
@ -470,11 +492,12 @@ public class Scheduler implements Runnable {
final Set<StreamIdentifier> streamsSynced = new HashSet<>(); final Set<StreamIdentifier> streamsSynced = new HashSet<>();
if (shouldSyncStreamsNow()) { if (shouldSyncStreamsNow()) {
final MetricsScope metricsScope = MetricsUtil.createMetricsWithOperation(metricsFactory, MULTI_STREAM_TRACKER); final MetricsScope metricsScope =
MetricsUtil.createMetricsWithOperation(metricsFactory, MULTI_STREAM_TRACKER);
try { try {
final Map<StreamIdentifier, StreamConfig> newStreamConfigMap = streamTracker.streamConfigList() final Map<StreamIdentifier, StreamConfig> newStreamConfigMap = streamTracker.streamConfigList().stream()
.stream().collect(Collectors.toMap(StreamConfig::streamIdentifier, Function.identity())); .collect(Collectors.toMap(StreamConfig::streamIdentifier, Function.identity()));
// This is done to ensure that we clean up the stale streams lingering in the lease table. // This is done to ensure that we clean up the stale streams lingering in the lease table.
// Only sync from lease table again if the currentStreamConfigMap and newStreamConfigMap contain // Only sync from lease table again if the currentStreamConfigMap and newStreamConfigMap contain
// different set of streams and Leader has not synced the leases yet // different set of streams and Leader has not synced the leases yet
@ -487,12 +510,16 @@ public class Scheduler implements Runnable {
final Set<StreamIdentifier> streamsFromLeaseTable = leaseTableLeases.stream() final Set<StreamIdentifier> streamsFromLeaseTable = leaseTableLeases.stream()
.map(lease -> StreamIdentifier.multiStreamInstance(lease.streamIdentifier())) .map(lease -> StreamIdentifier.multiStreamInstance(lease.streamIdentifier()))
.collect(Collectors.toSet()); .collect(Collectors.toSet());
// Remove stream from currentStreamConfigMap if this stream in not in the lease table and newStreamConfigMap. // Remove stream from currentStreamConfigMap if this stream in not in the lease table and
// newStreamConfigMap.
// This means that the leases have already been deleted by the last leader. // This means that the leases have already been deleted by the last leader.
currentStreamConfigMap.keySet().stream() currentStreamConfigMap.keySet().stream()
.filter(streamIdentifier -> !newStreamConfigMap.containsKey(streamIdentifier) .filter(streamIdentifier -> !newStreamConfigMap.containsKey(streamIdentifier)
&& !streamsFromLeaseTable.contains(streamIdentifier)).forEach(stream -> { && !streamsFromLeaseTable.contains(streamIdentifier))
log.info("Removing stream {} from currentStreamConfigMap due to not being active", stream); .forEach(stream -> {
log.info(
"Removing stream {} from currentStreamConfigMap due to not being active",
stream);
currentStreamConfigMap.remove(stream); currentStreamConfigMap.remove(stream);
staleStreamDeletionMap.remove(stream); staleStreamDeletionMap.remove(stream);
streamsSynced.add(stream); streamsSynced.add(stream);
@ -519,18 +546,21 @@ public class Scheduler implements Runnable {
staleStreamDeletionMap.putIfAbsent(streamIdentifier, Instant.now()); staleStreamDeletionMap.putIfAbsent(streamIdentifier, Instant.now());
} }
}; };
if (formerStreamsLeasesDeletionStrategy.leaseDeletionType() == FORMER_STREAMS_AUTO_DETECTION_DEFERRED_DELETION) { if (formerStreamsLeasesDeletionStrategy.leaseDeletionType()
== FORMER_STREAMS_AUTO_DETECTION_DEFERRED_DELETION) {
// Now, we are identifying the stale/old streams and enqueuing it for deferred deletion. // Now, we are identifying the stale/old streams and enqueuing it for deferred deletion.
// It is assumed that all the workers will always have the latest and consistent snapshot of streams // It is assumed that all the workers will always have the latest and consistent snapshot of streams
// from the multiStreamTracker. // from the multiStreamTracker.
// //
// The following streams transition state among two workers are NOT considered safe, where Worker 2, on // The following streams transition state among two workers are NOT considered safe, where Worker 2,
// on
// initialization learn about D from lease table and delete the leases for D, as it is not available // initialization learn about D from lease table and delete the leases for D, as it is not available
// in its latest MultiStreamTracker. // in its latest MultiStreamTracker.
// Worker 1 : A,B,C -> A,B,C,D (latest) // Worker 1 : A,B,C -> A,B,C,D (latest)
// Worker 2 : BOOTS_UP -> A,B,C (stale) // Worker 2 : BOOTS_UP -> A,B,C (stale)
// //
// The following streams transition state among two workers are NOT considered safe, where Worker 2 might // The following streams transition state among two workers are NOT considered safe, where Worker 2
// might
// end up deleting the leases for A and D and lose progress made so far. // end up deleting the leases for A and D and lose progress made so far.
// Worker 1 : A,B,C -> A,B,C,D (latest) // Worker 1 : A,B,C -> A,B,C,D (latest)
// Worker 2 : A,B,C -> B,C (stale/partial) // Worker 2 : A,B,C -> B,C (stale/partial)
@ -539,13 +569,16 @@ public class Scheduler implements Runnable {
// before attempting to delete it, we will be deferring the leases deletion based on the // before attempting to delete it, we will be deferring the leases deletion based on the
// defer time period. // defer time period.
currentStreamConfigMap.keySet().forEach(enqueueStreamLeaseDeletionOperation); currentStreamConfigMap.keySet().forEach(enqueueStreamLeaseDeletionOperation);
} else if (formerStreamsLeasesDeletionStrategy.leaseDeletionType() == StreamsLeasesDeletionType.PROVIDED_STREAMS_DEFERRED_DELETION) { } else if (formerStreamsLeasesDeletionStrategy.leaseDeletionType()
Optional.ofNullable(formerStreamsLeasesDeletionStrategy.streamIdentifiersForLeaseCleanup()).ifPresent( == StreamsLeasesDeletionType.PROVIDED_STREAMS_DEFERRED_DELETION) {
streamIdentifiers -> streamIdentifiers.forEach(enqueueStreamLeaseDeletionOperation)); Optional.ofNullable(formerStreamsLeasesDeletionStrategy.streamIdentifiersForLeaseCleanup())
.ifPresent(streamIdentifiers ->
streamIdentifiers.forEach(enqueueStreamLeaseDeletionOperation));
} else { } else {
// Remove the old/stale streams identified through the new and existing streams list, without // Remove the old/stale streams identified through the new and existing streams list, without
// cleaning up their leases. Disabling deprecated shard sync + lease cleanup through a flag. // cleaning up their leases. Disabling deprecated shard sync + lease cleanup through a flag.
Iterator<StreamIdentifier> currentSetOfStreamsIter = currentStreamConfigMap.keySet().iterator(); Iterator<StreamIdentifier> currentSetOfStreamsIter =
currentStreamConfigMap.keySet().iterator();
while (currentSetOfStreamsIter.hasNext()) { while (currentSetOfStreamsIter.hasNext()) {
StreamIdentifier streamIdentifier = currentSetOfStreamsIter.next(); StreamIdentifier streamIdentifier = currentSetOfStreamsIter.next();
if (!newStreamConfigMap.containsKey(streamIdentifier)) { if (!newStreamConfigMap.containsKey(streamIdentifier)) {
@ -553,13 +586,14 @@ public class Scheduler implements Runnable {
log.info( log.info(
"Found old/deleted stream : {}. Triggering shard sync. Removing from tracked active streams.", "Found old/deleted stream : {}. Triggering shard sync. Removing from tracked active streams.",
streamIdentifier); streamIdentifier);
ShardSyncTaskManager shardSyncTaskManager = createOrGetShardSyncTaskManager( ShardSyncTaskManager shardSyncTaskManager =
currentStreamConfigMap.get(streamIdentifier)); createOrGetShardSyncTaskManager(currentStreamConfigMap.get(streamIdentifier));
shardSyncTaskManager.submitShardSyncTask(); shardSyncTaskManager.submitShardSyncTask();
} else { } else {
log.info( log.info(
"Found old/deleted stream : {}. Removing from tracked active streams, but not cleaning up leases," "Found old/deleted stream : {}. Removing from tracked active streams, but not cleaning up leases,"
+ " as part of this workflow", streamIdentifier); + " as part of this workflow",
streamIdentifier);
} }
currentSetOfStreamsIter.remove(); currentSetOfStreamsIter.remove();
streamsSynced.add(streamIdentifier); streamsSynced.add(streamIdentifier);
@ -573,22 +607,24 @@ public class Scheduler implements Runnable {
// StreamIdentifiers are eligible for deletion only when the deferment period has elapsed and // StreamIdentifiers are eligible for deletion only when the deferment period has elapsed and
// the streamIdentifiersForLeaseCleanup are not present in the latest snapshot. // the streamIdentifiersForLeaseCleanup are not present in the latest snapshot.
final Map<Boolean, Set<StreamIdentifier>> staleStreamIdDeletionDecisionMap = final Map<Boolean, Set<StreamIdentifier>> staleStreamIdDeletionDecisionMap =
staleStreamDeletionMap.keySet().stream().collect( staleStreamDeletionMap.keySet().stream()
Collectors.partitioningBy(newStreamConfigMap::containsKey, Collectors.toSet())); .collect(
final Set<StreamIdentifier> staleStreamIdsToBeDeleted = staleStreamIdDeletionDecisionMap.get(false) Collectors.partitioningBy(newStreamConfigMap::containsKey, Collectors.toSet()));
.stream().filter(streamIdentifier -> final Set<StreamIdentifier> staleStreamIdsToBeDeleted =
Duration.between(staleStreamDeletionMap.get(streamIdentifier), Instant.now()) staleStreamIdDeletionDecisionMap.get(false).stream()
.toMillis() >= waitPeriodToDeleteOldStreams.toMillis()) .filter(streamIdentifier ->
.collect(Collectors.toSet()); Duration.between(staleStreamDeletionMap.get(streamIdentifier), Instant.now())
.toMillis()
>= waitPeriodToDeleteOldStreams.toMillis())
.collect(Collectors.toSet());
// These are the streams which are deleted in Kinesis and we encounter resource not found during // These are the streams which are deleted in Kinesis and we encounter resource not found during
// shardSyncTask. This is applicable in MultiStreamMode only, in case of SingleStreamMode, store will // shardSyncTask. This is applicable in MultiStreamMode only, in case of SingleStreamMode, store will
// not have any data. // not have any data.
// Filter streams based on newStreamConfigMap so that we don't override input to KCL in any case. // Filter streams based on newStreamConfigMap so that we don't override input to KCL in any case.
final Set<StreamIdentifier> deletedStreamSet = this.deletedStreamListProvider final Set<StreamIdentifier> deletedStreamSet =
.purgeAllDeletedStream() this.deletedStreamListProvider.purgeAllDeletedStream().stream()
.stream() .filter(streamIdentifier -> !newStreamConfigMap.containsKey(streamIdentifier))
.filter(streamIdentifier -> !newStreamConfigMap.containsKey(streamIdentifier)) .collect(Collectors.toSet());
.collect(Collectors.toSet());
if (deletedStreamSet.size() > 0) { if (deletedStreamSet.size() > 0) {
log.info("Stale streams to delete: {}", deletedStreamSet); log.info("Stale streams to delete: {}", deletedStreamSet);
staleStreamIdsToBeDeleted.addAll(deletedStreamSet); staleStreamIdsToBeDeleted.addAll(deletedStreamSet);
@ -603,18 +639,24 @@ public class Scheduler implements Runnable {
if (!staleStreamDeletionMap.isEmpty()) { if (!staleStreamDeletionMap.isEmpty()) {
log.warn( log.warn(
"Streams enqueued for deletion for lease table cleanup along with their scheduled time for deletion: {} ", "Streams enqueued for deletion for lease table cleanup along with their scheduled time for deletion: {} ",
staleStreamDeletionMap.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, staleStreamDeletionMap.entrySet().stream()
entry -> entry.getValue().plus(waitPeriodToDeleteOldStreams)))); .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue()
.plus(waitPeriodToDeleteOldStreams))));
} }
streamSyncWatch.reset().start(); streamSyncWatch.reset().start();
MetricsUtil.addCount(metricsScope, ACTIVE_STREAMS_COUNT, newStreamConfigMap.size(), MetricsLevel.SUMMARY); MetricsUtil.addCount(
MetricsUtil.addCount(metricsScope, PENDING_STREAMS_DELETION_COUNT, staleStreamDeletionMap.size(), metricsScope, ACTIVE_STREAMS_COUNT, newStreamConfigMap.size(), MetricsLevel.SUMMARY);
MetricsUtil.addCount(
metricsScope,
PENDING_STREAMS_DELETION_COUNT,
staleStreamDeletionMap.size(),
MetricsLevel.SUMMARY); MetricsLevel.SUMMARY);
MetricsUtil.addCount(metricsScope, NON_EXISTING_STREAM_DELETE_COUNT, deletedStreamSet.size(), MetricsUtil.addCount(
MetricsLevel.SUMMARY); metricsScope, NON_EXISTING_STREAM_DELETE_COUNT, deletedStreamSet.size(), MetricsLevel.SUMMARY);
MetricsUtil.addCount(metricsScope, DELETED_STREAMS_COUNT, deletedStreamsLeases.size(), MetricsLevel.SUMMARY); MetricsUtil.addCount(
metricsScope, DELETED_STREAMS_COUNT, deletedStreamsLeases.size(), MetricsLevel.SUMMARY);
} finally { } finally {
MetricsUtil.endScope(metricsScope); MetricsUtil.endScope(metricsScope);
} }
@ -624,8 +666,7 @@ public class Scheduler implements Runnable {
@VisibleForTesting @VisibleForTesting
boolean shouldSyncStreamsNow() { boolean shouldSyncStreamsNow() {
return isMultiStreamMode && return isMultiStreamMode && (streamSyncWatch.elapsed(TimeUnit.MILLISECONDS) > NEW_STREAM_CHECK_INTERVAL_MILLIS);
(streamSyncWatch.elapsed(TimeUnit.MILLISECONDS) > NEW_STREAM_CHECK_INTERVAL_MILLIS);
} }
@VisibleForTesting @VisibleForTesting
@ -642,7 +683,8 @@ public class Scheduler implements Runnable {
private List<MultiStreamLease> fetchMultiStreamLeases() private List<MultiStreamLease> fetchMultiStreamLeases()
throws DependencyException, ProvisionedThroughputException, InvalidStateException { throws DependencyException, ProvisionedThroughputException, InvalidStateException {
return (List<MultiStreamLease>) ((List) leaseCoordinator.leaseRefresher().listLeases()); return (List<MultiStreamLease>)
((List) leaseCoordinator.leaseRefresher().listLeases());
} }
private void removeStreamsFromStaleStreamsList(Set<StreamIdentifier> streamIdentifiers) { private void removeStreamsFromStaleStreamsList(Set<StreamIdentifier> streamIdentifiers) {
@ -659,8 +701,9 @@ public class Scheduler implements Runnable {
log.info("Deleting streams: {}", streamIdentifiers); log.info("Deleting streams: {}", streamIdentifiers);
final Set<StreamIdentifier> streamsSynced = new HashSet<>(); final Set<StreamIdentifier> streamsSynced = new HashSet<>();
final List<MultiStreamLease> leases = fetchMultiStreamLeases(); final List<MultiStreamLease> leases = fetchMultiStreamLeases();
final Map<String, List<MultiStreamLease>> streamIdToShardsMap = leases.stream().collect( final Map<String, List<MultiStreamLease>> streamIdToShardsMap = leases.stream()
Collectors.groupingBy(MultiStreamLease::streamIdentifier, Collectors.toCollection(ArrayList::new))); .collect(Collectors.groupingBy(
MultiStreamLease::streamIdentifier, Collectors.toCollection(ArrayList::new)));
for (StreamIdentifier streamIdentifier : streamIdentifiers) { for (StreamIdentifier streamIdentifier : streamIdentifiers) {
log.warn("Found old/deleted stream: {}. Directly deleting leases of this stream.", streamIdentifier); log.warn("Found old/deleted stream: {}. Directly deleting leases of this stream.", streamIdentifier);
// Removing streamIdentifier from this map so PSSM doesn't think there is a hole in the stream while // Removing streamIdentifier from this map so PSSM doesn't think there is a hole in the stream while
@ -690,7 +733,8 @@ public class Scheduler implements Runnable {
} catch (DependencyException | InvalidStateException | ProvisionedThroughputException e) { } catch (DependencyException | InvalidStateException | ProvisionedThroughputException e) {
log.error( log.error(
"Unable to delete stale stream lease {}. Skipping further deletions for this stream. Will retry later.", "Unable to delete stale stream lease {}. Skipping further deletions for this stream. Will retry later.",
lease.leaseKey(), e); lease.leaseKey(),
e);
return false; return false;
} }
} }
@ -760,8 +804,8 @@ public class Scheduler implements Runnable {
public CompletableFuture<Boolean> startGracefulShutdown() { public CompletableFuture<Boolean> startGracefulShutdown() {
synchronized (this) { synchronized (this) {
if (gracefulShutdownFuture == null) { if (gracefulShutdownFuture == null) {
gracefulShutdownFuture = gracefulShutdownCoordinator gracefulShutdownFuture =
.startGracefulShutdown(createGracefulShutdownCallable()); gracefulShutdownCoordinator.startGracefulShutdown(createGracefulShutdownCallable());
} }
} }
return gracefulShutdownFuture; return gracefulShutdownFuture;
@ -809,13 +853,15 @@ public class Scheduler implements Runnable {
// If there are no leases notification is already completed, but we still need to shutdown the worker. // If there are no leases notification is already completed, but we still need to shutdown the worker.
// //
this.shutdown(); this.shutdown();
return GracefulShutdownContext.builder().finalShutdownLatch(finalShutdownLatch).build(); return GracefulShutdownContext.builder()
.finalShutdownLatch(finalShutdownLatch)
.build();
} }
CountDownLatch shutdownCompleteLatch = new CountDownLatch(leases.size()); CountDownLatch shutdownCompleteLatch = new CountDownLatch(leases.size());
CountDownLatch notificationCompleteLatch = new CountDownLatch(leases.size()); CountDownLatch notificationCompleteLatch = new CountDownLatch(leases.size());
for (Lease lease : leases) { for (Lease lease : leases) {
ShutdownNotification shutdownNotification = new ShardConsumerShutdownNotification(leaseCoordinator, ShutdownNotification shutdownNotification = new ShardConsumerShutdownNotification(
lease, notificationCompleteLatch, shutdownCompleteLatch); leaseCoordinator, lease, notificationCompleteLatch, shutdownCompleteLatch);
ShardInfo shardInfo = DynamoDBLeaseCoordinator.convertLeaseToAssignment(lease); ShardInfo shardInfo = DynamoDBLeaseCoordinator.convertLeaseToAssignment(lease);
ShardConsumer consumer = shardInfoShardConsumerMap.get(shardInfo); ShardConsumer consumer = shardInfoShardConsumerMap.get(shardInfo);
if (consumer != null) { if (consumer != null) {
@ -929,9 +975,10 @@ public class Scheduler implements Runnable {
* Kinesis shard info * Kinesis shard info
* @return ShardConsumer for the shard * @return ShardConsumer for the shard
*/ */
ShardConsumer createOrGetShardConsumer(@NonNull final ShardInfo shardInfo, ShardConsumer createOrGetShardConsumer(
@NonNull final ShardRecordProcessorFactory shardRecordProcessorFactory, @NonNull final ShardInfo shardInfo,
@NonNull final LeaseCleanupManager leaseCleanupManager) { @NonNull final ShardRecordProcessorFactory shardRecordProcessorFactory,
@NonNull final LeaseCleanupManager leaseCleanupManager) {
ShardConsumer consumer = shardInfoShardConsumerMap.get(shardInfo); ShardConsumer consumer = shardInfoShardConsumerMap.get(shardInfo);
// Instantiate a new consumer if we don't have one, or the one we // Instantiate a new consumer if we don't have one, or the one we
// had was from an earlier // had was from an earlier
@ -948,20 +995,23 @@ public class Scheduler implements Runnable {
} }
private ShardSyncTaskManager createOrGetShardSyncTaskManager(StreamConfig streamConfig) { private ShardSyncTaskManager createOrGetShardSyncTaskManager(StreamConfig streamConfig) {
return streamToShardSyncTaskManagerMap.computeIfAbsent(streamConfig, s -> shardSyncTaskManagerProvider.apply(s)); return streamToShardSyncTaskManagerMap.computeIfAbsent(
streamConfig, s -> shardSyncTaskManagerProvider.apply(s));
} }
protected ShardConsumer buildConsumer(@NonNull final ShardInfo shardInfo, protected ShardConsumer buildConsumer(
@NonNull final ShardRecordProcessorFactory shardRecordProcessorFactory, @NonNull final ShardInfo shardInfo,
@NonNull final LeaseCleanupManager leaseCleanupManager) { @NonNull final ShardRecordProcessorFactory shardRecordProcessorFactory,
ShardRecordProcessorCheckpointer checkpointer = coordinatorConfig.coordinatorFactory().createRecordProcessorCheckpointer(shardInfo, @NonNull final LeaseCleanupManager leaseCleanupManager) {
checkpoint); ShardRecordProcessorCheckpointer checkpointer =
coordinatorConfig.coordinatorFactory().createRecordProcessorCheckpointer(shardInfo, checkpoint);
// The only case where streamName is not available will be when multistreamtracker not set. In this case, // The only case where streamName is not available will be when multistreamtracker not set. In this case,
// get the default stream name for the single stream application. // get the default stream name for the single stream application.
final StreamIdentifier streamIdentifier = getStreamIdentifier(shardInfo.streamIdentifierSerOpt()); final StreamIdentifier streamIdentifier = getStreamIdentifier(shardInfo.streamIdentifierSerOpt());
// Irrespective of single stream app or multi stream app, streamConfig should always be available. // Irrespective of single stream app or multi stream app, streamConfig should always be available.
// If we have a shardInfo, that is not present in currentStreamConfigMap for whatever reason, then return default stream config // If we have a shardInfo, that is not present in currentStreamConfigMap for whatever reason, then return
// default stream config
// to gracefully complete the reading. // to gracefully complete the reading.
StreamConfig streamConfig = currentStreamConfigMap.get(streamIdentifier); StreamConfig streamConfig = currentStreamConfigMap.get(streamIdentifier);
if (streamConfig == null) { if (streamConfig == null) {
@ -969,8 +1019,10 @@ public class Scheduler implements Runnable {
log.info("Created orphan {}", streamConfig); log.info("Created orphan {}", streamConfig);
} }
Validate.notNull(streamConfig, "StreamConfig should not be null"); Validate.notNull(streamConfig, "StreamConfig should not be null");
RecordsPublisher cache = retrievalConfig.retrievalFactory().createGetRecordsCache(shardInfo, streamConfig, metricsFactory); RecordsPublisher cache =
ShardConsumerArgument argument = new ShardConsumerArgument(shardInfo, retrievalConfig.retrievalFactory().createGetRecordsCache(shardInfo, streamConfig, metricsFactory);
ShardConsumerArgument argument = new ShardConsumerArgument(
shardInfo,
streamConfig.streamIdentifier(), streamConfig.streamIdentifier(),
leaseCoordinator, leaseCoordinator,
executorService, executorService,
@ -993,10 +1045,15 @@ public class Scheduler implements Runnable {
hierarchicalShardSyncerProvider.apply(streamConfig), hierarchicalShardSyncerProvider.apply(streamConfig),
metricsFactory, metricsFactory,
leaseCleanupManager, leaseCleanupManager,
schemaRegistryDecoder schemaRegistryDecoder);
); return new ShardConsumer(
return new ShardConsumer(cache, executorService, shardInfo, lifecycleConfig.logWarningForTaskAfterMillis(), cache,
argument, lifecycleConfig.taskExecutionListener(), lifecycleConfig.readTimeoutsToIgnoreBeforeWarning()); executorService,
shardInfo,
lifecycleConfig.logWarningForTaskAfterMillis(),
argument,
lifecycleConfig.taskExecutionListener(),
lifecycleConfig.readTimeoutsToIgnoreBeforeWarning());
} }
/** /**
@ -1030,16 +1087,16 @@ public class Scheduler implements Runnable {
*/ */
private void registerErrorHandlerForUndeliverableAsyncTaskExceptions() { private void registerErrorHandlerForUndeliverableAsyncTaskExceptions() {
RxJavaPlugins.setErrorHandler(t -> { RxJavaPlugins.setErrorHandler(t -> {
ExecutorStateEvent executorStateEvent = diagnosticEventFactory.executorStateEvent(executorService, ExecutorStateEvent executorStateEvent =
leaseCoordinator); diagnosticEventFactory.executorStateEvent(executorService, leaseCoordinator);
RejectedTaskEvent rejectedTaskEvent = diagnosticEventFactory.rejectedTaskEvent(executorStateEvent, t); RejectedTaskEvent rejectedTaskEvent = diagnosticEventFactory.rejectedTaskEvent(executorStateEvent, t);
rejectedTaskEvent.accept(diagnosticEventHandler); rejectedTaskEvent.accept(diagnosticEventHandler);
}); });
} }
private void logExecutorState() { private void logExecutorState() {
ExecutorStateEvent executorStateEvent = diagnosticEventFactory.executorStateEvent(executorService, ExecutorStateEvent executorStateEvent =
leaseCoordinator); diagnosticEventFactory.executorStateEvent(executorService, leaseCoordinator);
executorStateEvent.accept(diagnosticEventHandler); executorStateEvent.accept(diagnosticEventHandler);
} }
@ -1049,7 +1106,8 @@ public class Scheduler implements Runnable {
streamIdentifier = StreamIdentifier.multiStreamInstance(streamIdentifierString.get()); streamIdentifier = StreamIdentifier.multiStreamInstance(streamIdentifierString.get());
} else { } else {
Validate.isTrue(!isMultiStreamMode, "Should not be in MultiStream Mode"); Validate.isTrue(!isMultiStreamMode, "Should not be in MultiStream Mode");
streamIdentifier = this.currentStreamConfigMap.values().iterator().next().streamIdentifier(); streamIdentifier =
this.currentStreamConfigMap.values().iterator().next().streamIdentifier();
} }
Validate.notNull(streamIdentifier, "Stream identifier should not be empty"); Validate.notNull(streamIdentifier, "Stream identifier should not be empty");
return streamIdentifier; return streamIdentifier;
@ -1070,13 +1128,16 @@ public class Scheduler implements Runnable {
*/ */
private static StreamConfig withStreamArn( private static StreamConfig withStreamArn(
@NonNull final StreamConfig streamConfig, @NonNull final Region kinesisRegion) { @NonNull final StreamConfig streamConfig, @NonNull final Region kinesisRegion) {
Validate.isTrue(streamConfig.streamIdentifier().accountIdOptional().isPresent(), Validate.isTrue(
"accountId should not be empty"); streamConfig.streamIdentifier().accountIdOptional().isPresent(), "accountId should not be empty");
Validate.isTrue(streamConfig.streamIdentifier().streamCreationEpochOptional().isPresent(), Validate.isTrue(
streamConfig.streamIdentifier().streamCreationEpochOptional().isPresent(),
"streamCreationEpoch should not be empty"); "streamCreationEpoch should not be empty");
log.info("Constructing stream ARN for {} using the Kinesis client's configured region - {}.", log.info(
streamConfig.streamIdentifier(), kinesisRegion); "Constructing stream ARN for {} using the Kinesis client's configured region - {}.",
streamConfig.streamIdentifier(),
kinesisRegion);
final StreamIdentifier streamIdentifierWithArn = StreamIdentifier.multiStreamInstance( final StreamIdentifier streamIdentifierWithArn = StreamIdentifier.multiStreamInstance(
constructStreamArn( constructStreamArn(
@ -1106,13 +1167,22 @@ public class Scheduler implements Runnable {
@NonNull final StreamIdentifier streamIdentifier, @NonNull final StreamConfig streamConfig) { @NonNull final StreamIdentifier streamIdentifier, @NonNull final StreamConfig streamConfig) {
final Region kinesisRegion = getKinesisRegion(); final Region kinesisRegion = getKinesisRegion();
return super.put(streamIdentifier, streamConfig.streamIdentifier().streamArnOptional() return super.put(
.map(streamArn -> { streamIdentifier,
Validate.isTrue(kinesisRegion.id().equals(streamArn.region().get()), streamConfig
"The provided streamARN " + streamArn .streamIdentifier()
+ " does not match the Kinesis client's configured region - " + kinesisRegion); .streamArnOptional()
return streamConfig; .map(streamArn -> {
}).orElse(isMultiStreamMode ? withStreamArn(streamConfig, kinesisRegion) : streamConfig)); Validate.isTrue(
kinesisRegion
.id()
.equals(streamArn.region().get()),
"The provided streamARN " + streamArn
+ " does not match the Kinesis client's configured region - "
+ kinesisRegion);
return streamConfig;
})
.orElse(isMultiStreamMode ? withStreamArn(streamConfig, kinesisRegion) : streamConfig));
} }
} }

View file

@ -22,7 +22,6 @@ import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.common.util.concurrent.ThreadFactoryBuilder;
import lombok.Data; import lombok.Data;
import lombok.NonNull; import lombok.NonNull;
import software.amazon.kinesis.annotations.KinesisClientInternalApi; import software.amazon.kinesis.annotations.KinesisClientInternalApi;
@ -41,15 +40,16 @@ public class SchedulerCoordinatorFactory implements CoordinatorFactory {
*/ */
@Override @Override
public ExecutorService createExecutorService() { public ExecutorService createExecutorService() {
return new SchedulerThreadPoolExecutor( return new SchedulerThreadPoolExecutor(new ThreadFactoryBuilder()
new ThreadFactoryBuilder().setNameFormat("ShardRecordProcessor-%04d").build()); .setNameFormat("ShardRecordProcessor-%04d")
.build());
} }
static class SchedulerThreadPoolExecutor extends ThreadPoolExecutor { static class SchedulerThreadPoolExecutor extends ThreadPoolExecutor {
private static final long DEFAULT_KEEP_ALIVE = 60L; private static final long DEFAULT_KEEP_ALIVE = 60L;
SchedulerThreadPoolExecutor(ThreadFactory threadFactory) { SchedulerThreadPoolExecutor(ThreadFactory threadFactory) {
super(0, Integer.MAX_VALUE, DEFAULT_KEEP_ALIVE, TimeUnit.SECONDS, new SynchronousQueue<>(), super(0, Integer.MAX_VALUE, DEFAULT_KEEP_ALIVE, TimeUnit.SECONDS, new SynchronousQueue<>(), threadFactory);
threadFactory);
} }
} }
@ -57,8 +57,8 @@ public class SchedulerCoordinatorFactory implements CoordinatorFactory {
* {@inheritDoc} * {@inheritDoc}
*/ */
@Override @Override
public ShardRecordProcessorCheckpointer createRecordProcessorCheckpointer(@NonNull final ShardInfo shardInfo, public ShardRecordProcessorCheckpointer createRecordProcessorCheckpointer(
@NonNull final Checkpointer checkpoint) { @NonNull final ShardInfo shardInfo, @NonNull final Checkpointer checkpoint) {
return new ShardRecordProcessorCheckpointer(shardInfo, checkpoint); return new ShardRecordProcessorCheckpointer(shardInfo, checkpoint);
} }
} }

View file

@ -29,6 +29,5 @@ public interface WorkerStateChangeListener {
void onWorkerStateChange(WorkerState newState); void onWorkerStateChange(WorkerState newState);
default void onAllInitializationAttemptsFailed(Throwable e) { default void onAllInitializationAttemptsFailed(Throwable e) {}
}
} }

View file

@ -19,16 +19,16 @@ package software.amazon.kinesis.exceptions;
* is not found). * is not found).
*/ */
public class InvalidStateException extends KinesisClientLibNonRetryableException { public class InvalidStateException extends KinesisClientLibNonRetryableException {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
/** /**
* @param message provides more details about the cause and potential ways to debug/address. * @param message provides more details about the cause and potential ways to debug/address.
*/ */
public InvalidStateException(String message) { public InvalidStateException(String message) {
super(message); super(message);
} }
/** /**
* @param message provides more details about the cause and potential ways to debug/address. * @param message provides more details about the cause and potential ways to debug/address.
* @param e Cause of the exception * @param e Cause of the exception
@ -36,5 +36,4 @@ public class InvalidStateException extends KinesisClientLibNonRetryableException
public InvalidStateException(String message, Exception e) { public InvalidStateException(String message, Exception e) {
super(message, e); super(message, e);
} }
} }

View file

@ -15,21 +15,21 @@
package software.amazon.kinesis.exceptions; package software.amazon.kinesis.exceptions;
/** /**
* This is thrown when the Amazon Kinesis Client Library encounters issues talking to its dependencies * This is thrown when the Amazon Kinesis Client Library encounters issues talking to its dependencies
* (e.g. fetching data from Kinesis, DynamoDB table reads/writes, emitting metrics to CloudWatch). * (e.g. fetching data from Kinesis, DynamoDB table reads/writes, emitting metrics to CloudWatch).
* *
*/ */
public class KinesisClientLibDependencyException extends KinesisClientLibRetryableException { public class KinesisClientLibDependencyException extends KinesisClientLibRetryableException {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
/** /**
* @param message provides more details about the cause and potential ways to debug/address. * @param message provides more details about the cause and potential ways to debug/address.
*/ */
public KinesisClientLibDependencyException(String message) { public KinesisClientLibDependencyException(String message) {
super(message); super(message);
} }
/** /**
* @param message provides more details about the cause and potential ways to debug/address. * @param message provides more details about the cause and potential ways to debug/address.
* @param e Cause of the exception * @param e Cause of the exception
@ -37,5 +37,4 @@ public class KinesisClientLibDependencyException extends KinesisClientLibRetryab
public KinesisClientLibDependencyException(String message, Exception e) { public KinesisClientLibDependencyException(String message, Exception e) {
super(message, e); super(message, e);
} }
} }

View file

@ -25,7 +25,7 @@ public abstract class KinesisClientLibException extends Exception {
/** /**
* Constructor. * Constructor.
* *
* @param message Message of with details of the exception. * @param message Message of with details of the exception.
*/ */
public KinesisClientLibException(String message) { public KinesisClientLibException(String message) {
@ -34,12 +34,11 @@ public abstract class KinesisClientLibException extends Exception {
/** /**
* Constructor. * Constructor.
* *
* @param message Message with details of the exception. * @param message Message with details of the exception.
* @param cause Cause. * @param cause Cause.
*/ */
public KinesisClientLibException(String message, Throwable cause) { public KinesisClientLibException(String message, Throwable cause) {
super(message, cause); super(message, cause);
} }
} }

View file

@ -16,7 +16,7 @@ package software.amazon.kinesis.exceptions;
/** /**
* Non-retryable exceptions. Simply retrying the same request/operation is not expected to succeed. * Non-retryable exceptions. Simply retrying the same request/operation is not expected to succeed.
* *
*/ */
public abstract class KinesisClientLibNonRetryableException extends KinesisClientLibException { public abstract class KinesisClientLibNonRetryableException extends KinesisClientLibException {
@ -24,7 +24,7 @@ public abstract class KinesisClientLibNonRetryableException extends KinesisClien
/** /**
* Constructor. * Constructor.
* *
* @param message Message. * @param message Message.
*/ */
public KinesisClientLibNonRetryableException(String message) { public KinesisClientLibNonRetryableException(String message) {
@ -33,7 +33,7 @@ public abstract class KinesisClientLibNonRetryableException extends KinesisClien
/** /**
* Constructor. * Constructor.
* *
* @param message Message. * @param message Message.
* @param e Cause. * @param e Cause.
*/ */

View file

@ -22,7 +22,7 @@ public abstract class KinesisClientLibRetryableException extends RuntimeExceptio
/** /**
* Constructor. * Constructor.
* *
* @param message Message with details about the exception. * @param message Message with details about the exception.
*/ */
public KinesisClientLibRetryableException(String message) { public KinesisClientLibRetryableException(String message) {
@ -31,7 +31,7 @@ public abstract class KinesisClientLibRetryableException extends RuntimeExceptio
/** /**
* Constructor. * Constructor.
* *
* @param message Message with details about the exception. * @param message Message with details about the exception.
* @param e Cause. * @param e Cause.
*/ */

Some files were not shown because too many files have changed in this diff Show more