luciano/amazon-kinesis-client
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

run spotless:apply

Browse source
This commit is contained in:
Nakul Joshi 2024-05-20 11:26:15 -07:00 committed by nakulj
parent 409a8f13cd
commit 18fe49eed0
328 changed files with 14517 additions and 10319 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

216
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/coordinator/KinesisClientLibConfiguration.java
View file

@ -18,11 +18,9 @@ import java.util.Date;
import java.util.Optional;
import java.util.Set;
import org.apache.commons.lang3.Validate;
import com.google.common.collect.ImmutableSet;
import lombok.Getter;
import org.apache.commons.lang3.Validate;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.kinesis.checkpoint.ShardRecordProcessorCheckpointer;
import software.amazon.kinesis.common.InitialPositionInStream;
@ -119,14 +117,16 @@ public class KinesisClientLibConfiguration {
/**
* Metrics dimensions that always will be enabled regardless of the config provided by user.
*/
public static final Set<String> METRICS_ALWAYS_ENABLED_DIMENSIONS = ImmutableSet
.of(MetricsUtil.OPERATION_DIMENSION_NAME);
public static final Set<String> METRICS_ALWAYS_ENABLED_DIMENSIONS =
ImmutableSet.of(MetricsUtil.OPERATION_DIMENSION_NAME);
/**
* Allowed dimensions for CloudWatch metrics. By default, worker ID dimension will be disabled.
*/
public static final Set<String> DEFAULT_METRICS_ENABLED_DIMENSIONS = ImmutableSet.<String> builder()
.addAll(METRICS_ALWAYS_ENABLED_DIMENSIONS).add(MetricsUtil.SHARD_ID_DIMENSION_NAME).build();
public static final Set<String> DEFAULT_METRICS_ENABLED_DIMENSIONS = ImmutableSet.<String>builder()
.addAll(METRICS_ALWAYS_ENABLED_DIMENSIONS)
.add(MetricsUtil.SHARD_ID_DIMENSION_NAME)
.build();
/**
* Metrics dimensions that signify all possible dimensions.
@ -285,8 +285,8 @@ public class KinesisClientLibConfiguration {
* @param workerId
* Used to distinguish different workers/processes of a Kinesis application
*/
public KinesisClientLibConfiguration(String applicationName, String streamName,
AwsCredentialsProvider credentialsProvider, String workerId) {
public KinesisClientLibConfiguration(
String applicationName, String streamName, AwsCredentialsProvider credentialsProvider, String workerId) {
this(applicationName, streamName, credentialsProvider, credentialsProvider, credentialsProvider, workerId);
}
@ -308,16 +308,36 @@ public class KinesisClientLibConfiguration {
* @param workerId
* Used to distinguish different workers/processes of a Kinesis application
*/
public KinesisClientLibConfiguration(String applicationName, String streamName,
AwsCredentialsProvider kinesisCredentialsProvider, AwsCredentialsProvider dynamoDBCredentialsProvider,
AwsCredentialsProvider cloudWatchCredentialsProvider, String workerId) {
this(applicationName, streamName, null, null, DEFAULT_INITIAL_POSITION_IN_STREAM, kinesisCredentialsProvider,
dynamoDBCredentialsProvider, cloudWatchCredentialsProvider, DEFAULT_FAILOVER_TIME_MILLIS, workerId,
DEFAULT_MAX_RECORDS, DEFAULT_IDLETIME_BETWEEN_READS_MILLIS,
DEFAULT_DONT_CALL_PROCESS_RECORDS_FOR_EMPTY_RECORD_LIST, DEFAULT_PARENT_SHARD_POLL_INTERVAL_MILLIS,
DEFAULT_SHARD_SYNC_INTERVAL_MILLIS, DEFAULT_CLEANUP_LEASES_UPON_SHARDS_COMPLETION,
DEFAULT_TASK_BACKOFF_TIME_MILLIS, DEFAULT_METRICS_BUFFER_TIME_MILLIS, DEFAULT_METRICS_MAX_QUEUE_SIZE,
DEFAULT_VALIDATE_SEQUENCE_NUMBER_BEFORE_CHECKPOINTING, null, DEFAULT_SHUTDOWN_GRACE_MILLIS,
public KinesisClientLibConfiguration(
String applicationName,
String streamName,
AwsCredentialsProvider kinesisCredentialsProvider,
AwsCredentialsProvider dynamoDBCredentialsProvider,
AwsCredentialsProvider cloudWatchCredentialsProvider,
String workerId) {
this(
applicationName,
streamName,
null,
null,
DEFAULT_INITIAL_POSITION_IN_STREAM,
kinesisCredentialsProvider,
dynamoDBCredentialsProvider,
cloudWatchCredentialsProvider,
DEFAULT_FAILOVER_TIME_MILLIS,
workerId,
DEFAULT_MAX_RECORDS,
DEFAULT_IDLETIME_BETWEEN_READS_MILLIS,
DEFAULT_DONT_CALL_PROCESS_RECORDS_FOR_EMPTY_RECORD_LIST,
DEFAULT_PARENT_SHARD_POLL_INTERVAL_MILLIS,
DEFAULT_SHARD_SYNC_INTERVAL_MILLIS,
DEFAULT_CLEANUP_LEASES_UPON_SHARDS_COMPLETION,
DEFAULT_TASK_BACKOFF_TIME_MILLIS,
DEFAULT_METRICS_BUFFER_TIME_MILLIS,
DEFAULT_METRICS_MAX_QUEUE_SIZE,
DEFAULT_VALIDATE_SEQUENCE_NUMBER_BEFORE_CHECKPOINTING,
null,
DEFAULT_SHUTDOWN_GRACE_MILLIS,
DEFAULT_SCHEDULER_INITIALIZATION_BACKOFF_TIME_MILLIS);
}
@ -377,20 +397,53 @@ public class KinesisClientLibConfiguration {
*/
// CHECKSTYLE:IGNORE HiddenFieldCheck FOR NEXT 26 LINES
// CHECKSTYLE:IGNORE ParameterNumber FOR NEXT 26 LINES
public KinesisClientLibConfiguration(String applicationName, String streamName, String kinesisEndpoint,
InitialPositionInStream initialPositionInStream, AwsCredentialsProvider kinesisCredentialsProvider,
AwsCredentialsProvider dynamoDBCredentialsProvider, AwsCredentialsProvider cloudWatchCredentialsProvider,
long failoverTimeMillis, String workerId, int maxRecords, long idleTimeBetweenReadsInMillis,
boolean callProcessRecordsEvenForEmptyRecordList, long parentShardPollIntervalMillis,
long shardSyncIntervalMillis, boolean cleanupTerminatedShardsBeforeExpiry, long taskBackoffTimeMillis,
long metricsBufferTimeMillis, int metricsMaxQueueSize, boolean validateSequenceNumberBeforeCheckpointing,
String regionName, long shutdownGraceMillis, long schedulerInitializationBackoffTimeMillis) {
this(applicationName, streamName, kinesisEndpoint, null, initialPositionInStream, kinesisCredentialsProvider,
dynamoDBCredentialsProvider, cloudWatchCredentialsProvider, failoverTimeMillis, workerId, maxRecords,
idleTimeBetweenReadsInMillis, callProcessRecordsEvenForEmptyRecordList, parentShardPollIntervalMillis,
shardSyncIntervalMillis, cleanupTerminatedShardsBeforeExpiry, taskBackoffTimeMillis,
metricsBufferTimeMillis, metricsMaxQueueSize, validateSequenceNumberBeforeCheckpointing, regionName,
shutdownGraceMillis, schedulerInitializationBackoffTimeMillis);
public KinesisClientLibConfiguration(
String applicationName,
String streamName,
String kinesisEndpoint,
InitialPositionInStream initialPositionInStream,
AwsCredentialsProvider kinesisCredentialsProvider,
AwsCredentialsProvider dynamoDBCredentialsProvider,
AwsCredentialsProvider cloudWatchCredentialsProvider,
long failoverTimeMillis,
String workerId,
int maxRecords,
long idleTimeBetweenReadsInMillis,
boolean callProcessRecordsEvenForEmptyRecordList,
long parentShardPollIntervalMillis,
long shardSyncIntervalMillis,
boolean cleanupTerminatedShardsBeforeExpiry,
long taskBackoffTimeMillis,
long metricsBufferTimeMillis,
int metricsMaxQueueSize,
boolean validateSequenceNumberBeforeCheckpointing,
String regionName,
long shutdownGraceMillis,
long schedulerInitializationBackoffTimeMillis) {
this(
applicationName,
streamName,
kinesisEndpoint,
null,
initialPositionInStream,
kinesisCredentialsProvider,
dynamoDBCredentialsProvider,
cloudWatchCredentialsProvider,
failoverTimeMillis,
workerId,
maxRecords,
idleTimeBetweenReadsInMillis,
callProcessRecordsEvenForEmptyRecordList,
parentShardPollIntervalMillis,
shardSyncIntervalMillis,
cleanupTerminatedShardsBeforeExpiry,
taskBackoffTimeMillis,
metricsBufferTimeMillis,
metricsMaxQueueSize,
validateSequenceNumberBeforeCheckpointing,
regionName,
shutdownGraceMillis,
schedulerInitializationBackoffTimeMillis);
}
/**
@ -449,15 +502,30 @@ public class KinesisClientLibConfiguration {
*/
// CHECKSTYLE:IGNORE HiddenFieldCheck FOR NEXT 26 LINES
// CHECKSTYLE:IGNORE ParameterNumber FOR NEXT 26 LINES
public KinesisClientLibConfiguration(String applicationName, String streamName, String kinesisEndpoint,
String dynamoDBEndpoint, InitialPositionInStream initialPositionInStream,
AwsCredentialsProvider kinesisCredentialsProvider, AwsCredentialsProvider dynamoDBCredentialsProvider,
AwsCredentialsProvider cloudWatchCredentialsProvider, long failoverTimeMillis, String workerId,
int maxRecords, long idleTimeBetweenReadsInMillis, boolean callProcessRecordsEvenForEmptyRecordList,
long parentShardPollIntervalMillis, long shardSyncIntervalMillis,
boolean cleanupTerminatedShardsBeforeExpiry, long taskBackoffTimeMillis, long metricsBufferTimeMillis,
int metricsMaxQueueSize, boolean validateSequenceNumberBeforeCheckpointing, String regionName,
long shutdownGraceMillis, long schedulerInitializationBackoffTimeMillis) {
public KinesisClientLibConfiguration(
String applicationName,
String streamName,
String kinesisEndpoint,
String dynamoDBEndpoint,
InitialPositionInStream initialPositionInStream,
AwsCredentialsProvider kinesisCredentialsProvider,
AwsCredentialsProvider dynamoDBCredentialsProvider,
AwsCredentialsProvider cloudWatchCredentialsProvider,
long failoverTimeMillis,
String workerId,
int maxRecords,
long idleTimeBetweenReadsInMillis,
boolean callProcessRecordsEvenForEmptyRecordList,
long parentShardPollIntervalMillis,
long shardSyncIntervalMillis,
boolean cleanupTerminatedShardsBeforeExpiry,
long taskBackoffTimeMillis,
long metricsBufferTimeMillis,
int metricsMaxQueueSize,
boolean validateSequenceNumberBeforeCheckpointing,
String regionName,
long shutdownGraceMillis,
long schedulerInitializationBackoffTimeMillis) {
// Check following values are greater than zero
checkIsValuePositive("FailoverTimeMillis", failoverTimeMillis);
checkIsValuePositive("IdleTimeBetweenReadsInMillis", idleTimeBetweenReadsInMillis);
@ -495,8 +563,8 @@ public class KinesisClientLibConfiguration {
this.maxLeasesToStealAtOneTime = DEFAULT_MAX_LEASES_TO_STEAL_AT_ONE_TIME;
this.initialLeaseTableReadCapacity = DEFAULT_INITIAL_LEASE_TABLE_READ_CAPACITY;
this.initialLeaseTableWriteCapacity = DEFAULT_INITIAL_LEASE_TABLE_WRITE_CAPACITY;
this.initialPositionInStreamExtended = InitialPositionInStreamExtended
.newInitialPosition(initialPositionInStream);
this.initialPositionInStreamExtended =
InitialPositionInStreamExtended.newInitialPosition(initialPositionInStream);
this.skipShardSyncAtWorkerInitializationIfLeasesExist = DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST;
this.shardPrioritization = DEFAULT_SHARD_PRIORITIZATION;
this.recordsFetcherFactory = new SimpleRecordsFetcherFactory();
@ -559,15 +627,30 @@ public class KinesisClientLibConfiguration {
*/
// CHECKSTYLE:IGNORE HiddenFieldCheck FOR NEXT 26 LINES
// CHECKSTYLE:IGNORE ParameterNumber FOR NEXT 26 LINES
public KinesisClientLibConfiguration(String applicationName, String streamName, String kinesisEndpoint,
String dynamoDBEndpoint, InitialPositionInStream initialPositionInStream,
AwsCredentialsProvider kinesisCredentialsProvider, AwsCredentialsProvider dynamoDBCredentialsProvider,
AwsCredentialsProvider cloudWatchCredentialsProvider, long failoverTimeMillis, String workerId,
int maxRecords, long idleTimeBetweenReadsInMillis, boolean callProcessRecordsEvenForEmptyRecordList,
long parentShardPollIntervalMillis, long shardSyncIntervalMillis,
boolean cleanupTerminatedShardsBeforeExpiry, long taskBackoffTimeMillis, long metricsBufferTimeMillis,
int metricsMaxQueueSize, boolean validateSequenceNumberBeforeCheckpointing, String regionName,
RecordsFetcherFactory recordsFetcherFactory, long schedulerInitializationBackoffTimeMillis) {
public KinesisClientLibConfiguration(
String applicationName,
String streamName,
String kinesisEndpoint,
String dynamoDBEndpoint,
InitialPositionInStream initialPositionInStream,
AwsCredentialsProvider kinesisCredentialsProvider,
AwsCredentialsProvider dynamoDBCredentialsProvider,
AwsCredentialsProvider cloudWatchCredentialsProvider,
long failoverTimeMillis,
String workerId,
int maxRecords,
long idleTimeBetweenReadsInMillis,
boolean callProcessRecordsEvenForEmptyRecordList,
long parentShardPollIntervalMillis,
long shardSyncIntervalMillis,
boolean cleanupTerminatedShardsBeforeExpiry,
long taskBackoffTimeMillis,
long metricsBufferTimeMillis,
int metricsMaxQueueSize,
boolean validateSequenceNumberBeforeCheckpointing,
String regionName,
RecordsFetcherFactory recordsFetcherFactory,
long schedulerInitializationBackoffTimeMillis) {
// Check following values are greater than zero
checkIsValuePositive("FailoverTimeMillis", failoverTimeMillis);
checkIsValuePositive("IdleTimeBetweenReadsInMillis", idleTimeBetweenReadsInMillis);
@ -607,8 +690,8 @@ public class KinesisClientLibConfiguration {
this.maxLeasesToStealAtOneTime = DEFAULT_MAX_LEASES_TO_STEAL_AT_ONE_TIME;
this.initialLeaseTableReadCapacity = DEFAULT_INITIAL_LEASE_TABLE_READ_CAPACITY;
this.initialLeaseTableWriteCapacity = DEFAULT_INITIAL_LEASE_TABLE_WRITE_CAPACITY;
this.initialPositionInStreamExtended = InitialPositionInStreamExtended
.newInitialPosition(initialPositionInStream);
this.initialPositionInStreamExtended =
InitialPositionInStreamExtended.newInitialPosition(initialPositionInStream);
this.skipShardSyncAtWorkerInitializationIfLeasesExist = DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST;
this.shardPrioritization = DEFAULT_SHARD_PRIORITIZATION;
this.recordsFetcherFactory = recordsFetcherFactory;
@ -933,8 +1016,8 @@ public class KinesisClientLibConfiguration {
*/
public KinesisClientLibConfiguration withInitialPositionInStream(InitialPositionInStream initialPositionInStream) {
this.initialPositionInStream = initialPositionInStream;
this.initialPositionInStreamExtended = InitialPositionInStreamExtended
.newInitialPosition(initialPositionInStream);
this.initialPositionInStreamExtended =
InitialPositionInStreamExtended.newInitialPosition(initialPositionInStream);
return this;
}
@ -984,8 +1067,8 @@ public class KinesisClientLibConfiguration {
public KinesisClientLibConfiguration withMaxRecords(int maxRecords) {
checkIsValuePositive("MaxRecords", (long) maxRecords);
if (maxRecords > DEFAULT_MAX_RECORDS) {
throw new IllegalArgumentException(
"maxRecords must be less than or equal to " + DEFAULT_MAX_RECORDS + " but current value is " + maxRecords);
throw new IllegalArgumentException("maxRecords must be less than or equal to " + DEFAULT_MAX_RECORDS
+ " but current value is " + maxRecords);
}
this.maxRecords = maxRecords;
return this;
@ -1145,8 +1228,10 @@ public class KinesisClientLibConfiguration {
} else if (metricsEnabledDimensions.contains(MetricsScope.METRICS_DIMENSIONS_ALL)) {
this.metricsEnabledDimensions = METRICS_DIMENSIONS_ALL;
} else {
this.metricsEnabledDimensions = ImmutableSet.<String> builder().addAll(metricsEnabledDimensions)
.addAll(METRICS_ALWAYS_ENABLED_DIMENSIONS).build();
this.metricsEnabledDimensions = ImmutableSet.<String>builder()
.addAll(metricsEnabledDimensions)
.addAll(METRICS_ALWAYS_ENABLED_DIMENSIONS)
.build();
}
return this;
}
@ -1277,7 +1362,8 @@ public class KinesisClientLibConfiguration {
* @return this configuration object
*/
public KinesisClientLibConfiguration withMaxLeaseRenewalThreads(int maxLeaseRenewalThreads) {
Validate.isTrue(maxLeaseRenewalThreads > 2,
Validate.isTrue(
maxLeaseRenewalThreads > 2,
"The maximum number of lease renewal threads must be greater than or equal to 2.");
this.maxLeaseRenewalThreads = maxLeaseRenewalThreads;
@ -1337,7 +1423,8 @@ public class KinesisClientLibConfiguration {
* @return KinesisClientLibConfiguration
*/
public KinesisClientLibConfiguration withDataFetchingStrategy(String dataFetchingStrategy) {
this.recordsFetcherFactory.dataFetchingStrategy(DataFetchingStrategy.valueOf(dataFetchingStrategy.toUpperCase()));
this.recordsFetcherFactory.dataFetchingStrategy(
DataFetchingStrategy.valueOf(dataFetchingStrategy.toUpperCase()));
return this;
}
@ -1423,7 +1510,8 @@ public class KinesisClientLibConfiguration {
* Interval in milliseconds between retrying the scheduler initialization.
* @return
*/
public KinesisClientLibConfiguration withSchedulerInitializationBackoffTimeMillis(long schedulerInitializationBackoffTimeMillis) {
public KinesisClientLibConfiguration withSchedulerInitializationBackoffTimeMillis(
long schedulerInitializationBackoffTimeMillis) {
checkIsValuePositive("schedulerInitializationBackoffTimeMillis", schedulerInitializationBackoffTimeMillis);
this.schedulerInitializationBackoffTimeMillis = schedulerInitializationBackoffTimeMillis;
return this;

3
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/DrainChildSTDERRTask.java
View file

@ -23,8 +23,7 @@ import lombok.extern.slf4j.Slf4j;
*/
@Slf4j
class DrainChildSTDERRTask extends LineReaderTask<Boolean> {
DrainChildSTDERRTask() {
}
DrainChildSTDERRTask() {}
@Override
protected HandleLineResult<Boolean> handleLine(String line) {

3
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/DrainChildSTDOUTTask.java
View file

@ -37,8 +37,7 @@ import lombok.extern.slf4j.Slf4j;
*/
@Slf4j
class DrainChildSTDOUTTask extends LineReaderTask<Boolean> {
DrainChildSTDOUTTask() {
}
DrainChildSTDOUTTask() {}
@Override
protected HandleLineResult<Boolean> handleLine(String line) {

9
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/GetNextMessageTask.java
View file

@ -17,10 +17,9 @@ package software.amazon.kinesis.multilang;
import java.io.BufferedReader;
import java.io.IOException;
import software.amazon.kinesis.multilang.messages.Message;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j;
import software.amazon.kinesis.multilang.messages.Message;
/**
* Gets the next message off the STDOUT of the child process. Throws an exception if a message is not found before the
@ -71,8 +70,10 @@ class GetNextMessageTask extends LineReaderTask<Message> {
@Override
protected Message returnAfterException(Exception e) {
throw new RuntimeException("Encountered an error while reading a line from STDIN for shard " + getShardId()
+ " so won't be able to return a message.", e);
throw new RuntimeException(
"Encountered an error while reading a line from STDIN for shard " + getShardId()
+ " so won't be able to return a message.",
e);
}
@Override

4
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/LineReaderTask.java
View file

@ -41,8 +41,7 @@ abstract class LineReaderTask<T> implements Callable<T> {
private String shardId;
LineReaderTask() {
}
LineReaderTask() {}
/**
* Reads lines off the input stream until a return value is set, or an exception is encountered, or the end of the
@ -180,5 +179,4 @@ abstract class LineReaderTask<T> implements Callable<T> {
this.description = description;
return this;
}
}

22
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MessageReader.java
View file

@ -20,8 +20,8 @@ import java.io.InputStreamReader;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import software.amazon.kinesis.multilang.messages.Message;
import com.fasterxml.jackson.databind.ObjectMapper;
import software.amazon.kinesis.multilang.messages.Message;
/**
* Provides methods for interacting with the child process's STDOUT.
@ -48,8 +48,7 @@ class MessageReader {
/**
* Use the initialize methods after construction.
*/
MessageReader() {
}
MessageReader() {}
/**
* Returns a future which represents an attempt to read the next message in the child process's STDOUT. If the task
@ -95,13 +94,10 @@ class MessageReader {
* @param objectMapper The object mapper to decode messages.
* @param executorService An executor service to run tasks in.
*/
MessageReader initialize(InputStream stream,
String shardId,
ObjectMapper objectMapper,
ExecutorService executorService) {
return this.initialize(new BufferedReader(new InputStreamReader(stream)), shardId, objectMapper,
executorService);
MessageReader initialize(
InputStream stream, String shardId, ObjectMapper objectMapper, ExecutorService executorService) {
return this.initialize(
new BufferedReader(new InputStreamReader(stream)), shardId, objectMapper, executorService);
}
/**
@ -110,10 +106,8 @@ class MessageReader {
* @param objectMapper The object mapper to decode messages.
* @param executorService An executor service to run tasks in.
*/
MessageReader initialize(BufferedReader reader,
String shardId,
ObjectMapper objectMapper,
ExecutorService executorService) {
MessageReader initialize(
BufferedReader reader, String shardId, ObjectMapper objectMapper, ExecutorService executorService) {
this.reader = reader;
this.shardId = shardId;
this.objectMapper = objectMapper;

36
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MessageWriter.java
View file

@ -23,7 +23,6 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j;
import software.amazon.kinesis.lifecycle.events.InitializationInput;
import software.amazon.kinesis.lifecycle.events.LeaseLostInput;
@ -55,8 +54,7 @@ class MessageWriter {
/**
* Use initialize method after construction.
*/
MessageWriter() {
}
MessageWriter() {}
/**
* Writes the message then writes the line separator provided by the system. Flushes each message to guarantee it
@ -76,7 +74,10 @@ class MessageWriter {
*/
synchronized (writer) {
writer.write(message, 0, message.length());
writer.write(System.lineSeparator(), 0, System.lineSeparator().length());
writer.write(
System.lineSeparator(),
0,
System.lineSeparator().length());
writer.flush();
}
log.info("Message size == {} bytes for shard {}", message.getBytes().length, shardId);
@ -108,9 +109,9 @@ class MessageWriter {
String jsonText = objectMapper.writeValueAsString(message);
return writeMessageToOutput(jsonText);
} catch (IOException e) {
String errorMessage =
String.format("Encountered I/O error while writing %s action to subprocess", message.getClass()
.getSimpleName());
String errorMessage = String.format(
"Encountered I/O error while writing %s action to subprocess",
message.getClass().getSimpleName());
log.error(errorMessage, e);
throw new RuntimeException(errorMessage, e);
}
@ -175,8 +176,8 @@ class MessageWriter {
* @param throwable
* The exception that was thrown by a checkpoint attempt. Null if one didn't occur.
*/
Future<Boolean> writeCheckpointMessageWithError(String sequenceNumber, Long subSequenceNumber,
Throwable throwable) {
Future<Boolean> writeCheckpointMessageWithError(
String sequenceNumber, Long subSequenceNumber, Throwable throwable) {
return writeMessage(new CheckpointMessage(sequenceNumber, subSequenceNumber, throwable));
}
@ -207,12 +208,10 @@ class MessageWriter {
* @param objectMapper The object mapper to encode messages.
* @param executorService An executor service to run tasks in.
*/
MessageWriter initialize(OutputStream stream,
String shardId,
ObjectMapper objectMapper,
ExecutorService executorService) {
return this.initialize(new BufferedWriter(new OutputStreamWriter(stream)), shardId, objectMapper,
executorService);
MessageWriter initialize(
OutputStream stream, String shardId, ObjectMapper objectMapper, ExecutorService executorService) {
return this.initialize(
new BufferedWriter(new OutputStreamWriter(stream)), shardId, objectMapper, executorService);
}
/**
@ -221,15 +220,12 @@ class MessageWriter {
* @param objectMapper The object mapper to encode messages.
* @param executorService An executor service to run tasks in.
*/
MessageWriter initialize(BufferedWriter writer,
String shardId,
ObjectMapper objectMapper,
ExecutorService executorService) {
MessageWriter initialize(
BufferedWriter writer, String shardId, ObjectMapper objectMapper, ExecutorService executorService) {
this.writer = writer;
this.shardId = shardId;
this.objectMapper = objectMapper;
this.executorService = executorService;
return this;
}
}

39
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MultiLangDaemon.java
View file

@ -26,20 +26,18 @@ import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.joran.JoranConfigurator;
import ch.qos.logback.core.joran.spi.JoranException;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import lombok.Data;
import lombok.experimental.Accessors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.LoggerFactory;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.joran.JoranConfigurator;
import ch.qos.logback.core.joran.spi.JoranException;
import lombok.Data;
import lombok.experimental.Accessors;
import lombok.extern.slf4j.Slf4j;
import software.amazon.kinesis.coordinator.Scheduler;
/**
@ -75,11 +73,14 @@ public class MultiLangDaemon {
@Parameter
List<String> parameters = new ArrayList<>();
@Parameter(names = { "-p", "--properties-file" }, description = "Properties file to be used with the KCL")
@Parameter(
names = {"-p", "--properties-file"},
description = "Properties file to be used with the KCL")
String propertiesFile;
@Parameter(names = { "-l",
"--log-configuration" }, description = "File location of logback.xml to be override the default")
@Parameter(
names = {"-l", "--log-configuration"},
description = "File location of logback.xml to be override the default")
String logConfiguration;
}
@ -102,7 +103,8 @@ public class MultiLangDaemon {
}
JCommander buildJCommanderAndParseArgs(final MultiLangDaemonArguments arguments, final String[] args) {
JCommander jCommander = JCommander.newBuilder().programName("amazon-kinesis-client MultiLangDaemon")
JCommander jCommander = JCommander.newBuilder()
.programName("amazon-kinesis-client MultiLangDaemon")
.addObject(arguments)
.build();
jCommander.parse(args);
@ -128,8 +130,8 @@ public class MultiLangDaemon {
}
}
void configureLogging(final String logConfiguration, final LoggerContext loggerContext,
final JoranConfigurator configurator) {
void configureLogging(
final String logConfiguration, final LoggerContext loggerContext, final JoranConfigurator configurator) {
loggerContext.reset();
try (InputStream inputStream = FileUtils.openInputStream(new File(logConfiguration))) {
configurator.setContext(loggerContext);
@ -146,9 +148,8 @@ public class MultiLangDaemon {
if (arguments.parameters.size() == 1) {
propertiesFile = arguments.parameters.get(0);
} else {
throw new RuntimeException(
"Expected a single argument, but found multiple arguments. Arguments: "
+ String.join(", ", arguments.parameters));
throw new RuntimeException("Expected a single argument, but found multiple arguments. Arguments: "
+ String.join(", ", arguments.parameters));
}
}

34
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MultiLangDaemonConfig.java
View file

@ -26,10 +26,9 @@ import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import software.amazon.kinesis.multilang.config.KinesisClientLibConfigurator;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import lombok.extern.slf4j.Slf4j;
import software.amazon.kinesis.multilang.config.KinesisClientLibConfigurator;
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
import software.amazon.kinesis.retrieval.RetrievalConfig;
@ -94,8 +93,9 @@ public class MultiLangDaemonConfig {
* @throws IllegalArgumentException
* Thrown when the contents of the properties file are not as expected.
*/
public MultiLangDaemonConfig(String propertiesFile, ClassLoader classLoader,
KinesisClientLibConfigurator configurator) throws IOException, IllegalArgumentException {
public MultiLangDaemonConfig(
String propertiesFile, ClassLoader classLoader, KinesisClientLibConfigurator configurator)
throws IOException, IllegalArgumentException {
Properties properties = loadProperties(classLoader, propertiesFile);
if (!validateProperties(properties)) {
throw new IllegalArgumentException(
@ -107,11 +107,14 @@ public class MultiLangDaemonConfig {
multiLangDaemonConfiguration = configurator.getConfiguration(properties);
executorService = buildExecutorService(properties);
recordProcessorFactory = new MultiLangRecordProcessorFactory(executableName, executorService,
multiLangDaemonConfiguration);
recordProcessorFactory =
new MultiLangRecordProcessorFactory(executableName, executorService, multiLangDaemonConfiguration);
log.info("Running {} to process stream {} with executable {}", multiLangDaemonConfiguration.getApplicationName(),
multiLangDaemonConfiguration.getStreamName(), executableName);
log.info(
"Running {} to process stream {} with executable {}",
multiLangDaemonConfiguration.getApplicationName(),
multiLangDaemonConfiguration.getStreamName(),
executableName);
prepare(processingLanguage);
}
@ -138,7 +141,7 @@ public class MultiLangDaemonConfig {
}
log.info("MultiLangDaemon is adding the following fields to the User Agent: {}", userAgent.toString());
// multiLangDaemonConfiguration.withUserAgent(userAgent.toString());
// multiLangDaemonConfiguration.withUserAgent(userAgent.toString());
}
private static Properties loadProperties(ClassLoader classLoader, String propertiesFileName) throws IOException {
@ -181,12 +184,17 @@ public class MultiLangDaemonConfig {
log.debug("Value for {} property is {}", PROP_MAX_ACTIVE_THREADS, maxActiveThreads);
if (maxActiveThreads <= 0) {
log.info("Using a cached thread pool.");
return new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, new SynchronousQueue<>(),
builder.build());
return new ThreadPoolExecutor(
0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, new SynchronousQueue<>(), builder.build());
} else {
log.info("Using a fixed thread pool with {} max active threads.", maxActiveThreads);
return new ThreadPoolExecutor(maxActiveThreads, maxActiveThreads, 0L, TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<>(), builder.build());
return new ThreadPoolExecutor(
maxActiveThreads,
maxActiveThreads,
0L,
TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<>(),
builder.build());
}
}

60
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MultiLangProtocol.java
View file

@ -61,8 +61,11 @@ class MultiLangProtocol {
* @param initializationInput
* information about the shard this processor is starting to process
*/
MultiLangProtocol(MessageReader messageReader, MessageWriter messageWriter,
InitializationInput initializationInput, MultiLangDaemonConfiguration configuration) {
MultiLangProtocol(
MessageReader messageReader,
MessageWriter messageWriter,
InitializationInput initializationInput,
MultiLangDaemonConfiguration configuration) {
this.messageReader = messageReader;
this.messageWriter = messageWriter;
this.initializationInput = initializationInput;
@ -82,7 +85,6 @@ class MultiLangProtocol {
*/
Future<Boolean> writeFuture = messageWriter.writeInitializeMessage(initializationInput);
return waitForStatusMessage(InitializeMessage.ACTION, null, writeFuture);
}
/**
@ -115,7 +117,9 @@ class MultiLangProtocol {
* @return
*/
boolean shardEnded(ShardEndedInput shardEndedInput) {
return waitForStatusMessage(ShardEndedMessage.ACTION, shardEndedInput.checkpointer(),
return waitForStatusMessage(
ShardEndedMessage.ACTION,
shardEndedInput.checkpointer(),
messageWriter.writeShardEndedMessage(shardEndedInput));
}
@ -147,8 +151,8 @@ class MultiLangProtocol {
* The writing task.
* @return Whether or not this operation succeeded.
*/
private boolean waitForStatusMessage(String action, RecordProcessorCheckpointer checkpointer,
Future<Boolean> writeFuture) {
private boolean waitForStatusMessage(
String action, RecordProcessorCheckpointer checkpointer, Future<Boolean> writeFuture) {
boolean statusWasCorrect = waitForStatusMessage(action, checkpointer);
// Examine whether or not we failed somewhere along the line.
@ -194,7 +198,7 @@ class MultiLangProtocol {
return false;
}
statusMessage = message.filter(m -> m instanceof StatusMessage).map(m -> (StatusMessage) m );
statusMessage = message.filter(m -> m instanceof StatusMessage).map(m -> (StatusMessage) m);
}
return this.validateStatusMessage(statusMessage.get(), action);
}
@ -207,13 +211,17 @@ class MultiLangProtocol {
try {
return Optional.of(fm.get());
} catch (InterruptedException e) {
log.error("Interrupted while waiting for {} message for shard {}", action,
initializationInput.shardId(), e);
log.error(
"Interrupted while waiting for {} message for shard {}", action, initializationInput.shardId(), e);
} catch (ExecutionException e) {
log.error("Failed to get status message for {} action for shard {}", action,
initializationInput.shardId(), e);
log.error(
"Failed to get status message for {} action for shard {}",
action,
initializationInput.shardId(),
e);
} catch (TimeoutException e) {
log.error("Timedout to get status message for {} action for shard {}. Terminating...",
log.error(
"Timedout to get status message for {} action for shard {}. Terminating...",
action,
initializationInput.shardId(),
e);
@ -240,11 +248,14 @@ class MultiLangProtocol {
* @return Whether or not this operation succeeded.
*/
private boolean validateStatusMessage(StatusMessage statusMessage, String action) {
log.info("Received response {} from subprocess while waiting for {}"
+ " while processing shard {}", statusMessage, action, initializationInput.shardId());
return !(statusMessage == null || statusMessage.getResponseFor() == null || !statusMessage.getResponseFor()
.equals(action));
log.info(
"Received response {} from subprocess while waiting for {}" + " while processing shard {}",
statusMessage,
action,
initializationInput.shardId());
return !(statusMessage == null
|| statusMessage.getResponseFor() == null
|| !statusMessage.getResponseFor().equals(action));
}
/**
@ -274,13 +285,12 @@ class MultiLangProtocol {
}
return this.messageWriter.writeCheckpointMessageWithError(sequenceNumber, subSequenceNumber, null);
} else {
String message =
String.format("Was asked to checkpoint at %s but no checkpointer was provided for shard %s",
sequenceNumber, initializationInput.shardId());
String message = String.format(
"Was asked to checkpoint at %s but no checkpointer was provided for shard %s",
sequenceNumber, initializationInput.shardId());
log.error(message);
return this.messageWriter.writeCheckpointMessageWithError(sequenceNumber, subSequenceNumber,
new InvalidStateException(
message));
return this.messageWriter.writeCheckpointMessageWithError(
sequenceNumber, subSequenceNumber, new InvalidStateException(message));
}
} catch (Throwable t) {
return this.messageWriter.writeCheckpointMessageWithError(sequenceNumber, subSequenceNumber, t);
@ -288,8 +298,8 @@ class MultiLangProtocol {
}
private String logCheckpointMessage(String sequenceNumber, Long subSequenceNumber) {
return String.format("Attempting to checkpoint shard %s @ sequence number %s, and sub sequence number %s",
return String.format(
"Attempting to checkpoint shard %s @ sequence number %s, and sub sequence number %s",
initializationInput.shardId(), sequenceNumber, subSequenceNumber);
}
}

18
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MultiLangRecordProcessorFactory.java
View file

@ -17,11 +17,10 @@ package software.amazon.kinesis.multilang;
import java.util.concurrent.ExecutorService;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j;
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
import software.amazon.kinesis.processor.ShardRecordProcessorFactory;
import software.amazon.kinesis.processor.ShardRecordProcessor;
import software.amazon.kinesis.processor.ShardRecordProcessorFactory;
/**
* Creates {@link MultiLangShardRecordProcessor}'s.
@ -43,8 +42,8 @@ public class MultiLangRecordProcessorFactory implements ShardRecordProcessorFact
* @param command The command that will do processing for this factory's record processors.
* @param executorService An executor service to use while processing inputs and outputs of the child process.
*/
public MultiLangRecordProcessorFactory(String command, ExecutorService executorService,
MultiLangDaemonConfiguration configuration) {
public MultiLangRecordProcessorFactory(
String command, ExecutorService executorService, MultiLangDaemonConfiguration configuration) {
this(command, executorService, new ObjectMapper(), configuration);
}
@ -53,8 +52,11 @@ public class MultiLangRecordProcessorFactory implements ShardRecordProcessorFact
* @param executorService An executor service to use while processing inputs and outputs of the child process.
* @param objectMapper An object mapper used to convert messages to json to be written to the child process
*/
public MultiLangRecordProcessorFactory(String command, ExecutorService executorService, ObjectMapper objectMapper,
MultiLangDaemonConfiguration configuration) {
public MultiLangRecordProcessorFactory(
String command,
ExecutorService executorService,
ObjectMapper objectMapper,
MultiLangDaemonConfiguration configuration) {
this.command = command;
this.commandArray = command.split(COMMAND_DELIMETER_REGEX);
this.executorService = executorService;
@ -68,8 +70,8 @@ public class MultiLangRecordProcessorFactory implements ShardRecordProcessorFact
/*
* Giving ProcessBuilder the command as an array of Strings allows users to specify command line arguments.
*/
return new MultiLangShardRecordProcessor(new ProcessBuilder(commandArray), executorService, this.objectMapper,
this.configuration);
return new MultiLangShardRecordProcessor(
new ProcessBuilder(commandArray), executorService, this.objectMapper, this.configuration);
}
String[] getCommandArray() {

38
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MultiLangShardRecordProcessor.java
View file

@ -22,7 +22,6 @@ import java.util.concurrent.Future;
import java.util.function.Function;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j;
import software.amazon.kinesis.lifecycle.events.InitializationInput;
import software.amazon.kinesis.lifecycle.events.LeaseLostInput;
@ -156,8 +155,10 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor {
if (ProcessState.ACTIVE.equals(this.state)) {
stopProcessing("Encountered an error while trying to shutdown child process", t);
} else {
stopProcessing("Encountered an error during shutdown,"
+ " but it appears the processor has already been shutdown", t);
stopProcessing(
"Encountered an error during shutdown,"
+ " but it appears the processor has already been shutdown",
t);
}
}
}
@ -166,7 +167,8 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor {
* Used to tell whether the processor has been shutdown already.
*/
private enum ProcessState {
ACTIVE, SHUTDOWN
ACTIVE,
SHUTDOWN
}
/**
@ -179,10 +181,19 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor {
* @param objectMapper
* An obejct mapper.
*/
MultiLangShardRecordProcessor(ProcessBuilder processBuilder, ExecutorService executorService,
ObjectMapper objectMapper, MultiLangDaemonConfiguration configuration) {
this(processBuilder, executorService, objectMapper, new MessageWriter(), new MessageReader(),
new DrainChildSTDERRTask(), configuration);
MultiLangShardRecordProcessor(
ProcessBuilder processBuilder,
ExecutorService executorService,
ObjectMapper objectMapper,
MultiLangDaemonConfiguration configuration) {
this(
processBuilder,
executorService,
objectMapper,
new MessageWriter(),
new MessageReader(),
new DrainChildSTDERRTask(),
configuration);
}
/**
@ -201,9 +212,14 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor {
* @param readSTDERRTask
* Error reader to read from child process's stderr
*/
MultiLangShardRecordProcessor(ProcessBuilder processBuilder, ExecutorService executorService, ObjectMapper objectMapper,
MessageWriter messageWriter, MessageReader messageReader, DrainChildSTDERRTask readSTDERRTask,
MultiLangDaemonConfiguration configuration) {
MultiLangShardRecordProcessor(
ProcessBuilder processBuilder,
ExecutorService executorService,
ObjectMapper objectMapper,
MessageWriter messageWriter,
MessageReader messageReader,
DrainChildSTDERRTask readSTDERRTask,
MultiLangDaemonConfiguration configuration) {
this.executorService = executorService;
this.processBuilder = processBuilder;
this.objectMapper = objectMapper;

3
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/NestedPropertyKey.java
View file

@ -19,7 +19,6 @@ import java.util.Map;
import com.amazonaws.regions.Regions;
import com.google.common.base.CaseFormat;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
@ -91,7 +90,6 @@ public enum NestedPropertyKey {
processor.acceptExternalId(externalId);
}
},
;
/**
@ -141,5 +139,4 @@ public enum NestedPropertyKey {
}
}
}
}

1
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/NestedPropertyProcessor.java
View file

@ -50,5 +50,4 @@ public interface NestedPropertyProcessor {
* @param externalId external id used in the service call used to retrieve session credentials
*/
void acceptExternalId(String externalId);
}

11
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/KclSTSAssumeRoleSessionCredentialsProvider.java
View file

@ -24,7 +24,6 @@ import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.securitytoken.AWSSecurityTokenService;
import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient;
import software.amazon.kinesis.multilang.NestedPropertyKey;
import software.amazon.kinesis.multilang.NestedPropertyProcessor;
@ -47,8 +46,8 @@ public class KclSTSAssumeRoleSessionCredentialsProvider
this(params[0], params[1], Arrays.copyOfRange(params, 2, params.length));
}
public KclSTSAssumeRoleSessionCredentialsProvider(final String roleArn, final String roleSessionName,
final String... params) {
public KclSTSAssumeRoleSessionCredentialsProvider(
final String roleArn, final String roleSessionName, final String... params) {
builder = new Builder(roleArn, roleSessionName);
NestedPropertyKey.parse(this, params);
provider = builder.build();
@ -75,9 +74,8 @@ public class KclSTSAssumeRoleSessionCredentialsProvider
@Override
public void acceptEndpointRegion(final Regions region) {
final AWSSecurityTokenService stsClient = AWSSecurityTokenServiceClient.builder()
.withRegion(region)
.build();
final AWSSecurityTokenService stsClient =
AWSSecurityTokenServiceClient.builder().withRegion(region).build();
builder.withStsClient(stsClient);
}
@ -85,5 +83,4 @@ public class KclSTSAssumeRoleSessionCredentialsProvider
public void acceptExternalId(final String externalId) {
builder.withExternalId(externalId);
}
}

35
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoder.java
View file

@ -37,8 +37,7 @@ class AWSCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode
/**
* Constructor.
*/
AWSCredentialsProviderPropertyValueDecoder() {
}
AWSCredentialsProviderPropertyValueDecoder() {}
/**
* Get AWSCredentialsProvider property.
@ -104,9 +103,8 @@ class AWSCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode
if (provider == null) {
// attempt to invoke a public varargs/array constructor of FooClass(String[])
provider = constructProvider(providerName, () ->
clazz.getConstructor(String[].class).newInstance((Object) varargs)
);
provider = constructProvider(providerName, () -> clazz.getConstructor(String[].class)
.newInstance((Object) varargs));
}
}
@ -138,24 +136,26 @@ class AWSCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode
private static List<String> getPossibleFullClassNames(final String provider) {
return Stream.of(
// Customer provides a short name of common providers in com.amazonaws.auth package
// (e.g., any classes implementing the AWSCredentialsProvider interface)
// @see http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/AWSCredentialsProvider.html
"com.amazonaws.auth.",
// Customer provides a short name of common providers in com.amazonaws.auth package
// (e.g., any classes implementing the AWSCredentialsProvider interface)
// @see
// http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/AWSCredentialsProvider.html
"com.amazonaws.auth.",
// Customer provides a short name of a provider offered by this multi-lang package
"software.amazon.kinesis.multilang.auth.",
// Customer provides a short name of a provider offered by this multi-lang package
"software.amazon.kinesis.multilang.auth.",
// Customer provides a fully-qualified provider name, or a custom credentials provider
// (e.g., com.amazonaws.auth.ClasspathFileCredentialsProvider, org.mycompany.FooProvider)
""
).map(prefix -> prefix + provider).collect(Collectors.toList());
// Customer provides a fully-qualified provider name, or a custom credentials provider
// (e.g., com.amazonaws.auth.ClasspathFileCredentialsProvider, org.mycompany.FooProvider)
"")
.map(prefix -> prefix + provider)
.collect(Collectors.toList());
}
@FunctionalInterface
private interface CredentialsProviderConstructor<T extends AWSCredentialsProvider> {
T construct() throws IllegalAccessException, InstantiationException,
InvocationTargetException, NoSuchMethodException;
T construct()
throws IllegalAccessException, InstantiationException, InvocationTargetException, NoSuchMethodException;
}
/**
@ -179,5 +179,4 @@ class AWSCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode
}
return null;
}
}

26
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/BuilderDynaBean.java
View file

@ -31,7 +31,7 @@ import org.apache.commons.lang3.StringUtils;
public class BuilderDynaBean implements DynaBean {
private static final String[] CLASS_NAME_JOINERS = { ClassUtils.PACKAGE_SEPARATOR, ClassUtils.INNER_CLASS_SEPARATOR };
private static final String[] CLASS_NAME_JOINERS = {ClassUtils.PACKAGE_SEPARATOR, ClassUtils.INNER_CLASS_SEPARATOR};
static final String NO_MAP_ACCESS_SUPPORT = "Map access isn't supported";
private Class<?> destinedClass;
@ -51,13 +51,19 @@ public class BuilderDynaBean implements DynaBean {
this(destinedClass, convertUtilsBean, null, Arrays.asList(classPrefixSearchList));
}
public BuilderDynaBean(Class<?> destinedClass, ConvertUtilsBean convertUtilsBean,
Function<String, ?> emptyPropertyHandler, String... classPrefixSearchList) {
public BuilderDynaBean(
Class<?> destinedClass,
ConvertUtilsBean convertUtilsBean,
Function<String, ?> emptyPropertyHandler,
String... classPrefixSearchList) {
this(destinedClass, convertUtilsBean, emptyPropertyHandler, Arrays.asList(classPrefixSearchList));
}
public BuilderDynaBean(Class<?> destinedClass, ConvertUtilsBean convertUtilsBean,
Function<String, ?> emptyPropertyHandler, List<String> classPrefixSearchList) {
public BuilderDynaBean(
Class<?> destinedClass,
ConvertUtilsBean convertUtilsBean,
Function<String, ?> emptyPropertyHandler,
List<String> classPrefixSearchList) {
this.convertUtilsBean = convertUtilsBean;
this.classPrefixSearchList = classPrefixSearchList;
this.emptyPropertyHandler = emptyPropertyHandler;
@ -102,7 +108,6 @@ public class BuilderDynaBean implements DynaBean {
// Ignored
//
}
}
}
}
@ -214,8 +219,10 @@ public class BuilderDynaBean implements DynaBean {
validateCanBuildOrCreate();
List<TypeTag> types = dynaBeanBuilderSupport.getProperty(name);
if (types.size() > 1) {
Optional<TypeTag> arrayType = types.stream().filter(t -> t.type.isArray()).findFirst();
return arrayType.map(t -> new DynaProperty(name, t.type, t.type.getComponentType()))
Optional<TypeTag> arrayType =
types.stream().filter(t -> t.type.isArray()).findFirst();
return arrayType
.map(t -> new DynaProperty(name, t.type, t.type.getComponentType()))
.orElseGet(() -> new DynaProperty(name));
} else {
TypeTag type = types.get(0);
@ -232,7 +239,8 @@ public class BuilderDynaBean implements DynaBean {
@Override
public DynaProperty[] getDynaProperties() {
validateCanBuildOrCreate();
return dynaBeanBuilderSupport.getPropertyNames().stream().map(this::getDynaProperty)
return dynaBeanBuilderSupport.getPropertyNames().stream()
.map(this::getDynaProperty)
.toArray(DynaProperty[]::new);
}

16
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/ConfigurationSettableUtils.java
View file

@ -23,13 +23,11 @@ import java.util.Map;
import java.util.Optional;
import java.util.Set;
import com.google.common.base.Defaults;
import lombok.NonNull;
import org.apache.commons.lang3.ClassUtils;
import org.apache.commons.lang3.StringUtils;
import com.google.common.base.Defaults;
import lombok.NonNull;
public class ConfigurationSettableUtils {
public static <T> T resolveFields(@NonNull Object source, @NonNull T configObject) {
@ -40,8 +38,8 @@ public class ConfigurationSettableUtils {
return configObject;
}
public static void resolveFields(Object source, Map<Class<?>, Object> configObjects, Set<Class<?>> restrictTo,
Set<Class<?>> skipIf) {
public static void resolveFields(
Object source, Map<Class<?>, Object> configObjects, Set<Class<?>> restrictTo, Set<Class<?>> skipIf) {
for (Field field : source.getClass().getDeclaredFields()) {
for (ConfigurationSettable b : field.getAnnotationsByType(ConfigurationSettable.class)) {
if (restrictTo != null && !restrictTo.contains(b.configurationClass())) {
@ -70,9 +68,11 @@ public class ConfigurationSettableUtils {
value = Optional.of(value);
}
if (ClassUtils.isPrimitiveOrWrapper(value.getClass())) {
Class<?> primitiveType = field.getType().isPrimitive() ? field.getType()
Class<?> primitiveType = field.getType().isPrimitive()
? field.getType()
: ClassUtils.wrapperToPrimitive(field.getType());
Class<?> wrapperType = !field.getType().isPrimitive() ? field.getType()
Class<?> wrapperType = !field.getType().isPrimitive()
? field.getType()
: ClassUtils.primitiveToWrapper(field.getType());
try {

4
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/DatePropertyValueDecoder.java
View file

@ -26,8 +26,7 @@ public class DatePropertyValueDecoder implements IPropertyValueDecoder<Date> {
/**
* Constructor.
*/
DatePropertyValueDecoder() {
}
DatePropertyValueDecoder() {}
/**
* @param value property value as String
@ -49,5 +48,4 @@ public class DatePropertyValueDecoder implements IPropertyValueDecoder<Date> {
public List<Class<Date>> getSupportedTypes() {
return Arrays.asList(Date.class);
}
}

47
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/DynaBeanBuilderSupport.java
View file

@ -29,11 +29,10 @@ import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Supplier;
import org.apache.commons.beanutils.ConvertUtilsBean;
import org.apache.commons.lang3.ClassUtils;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
import org.apache.commons.beanutils.ConvertUtilsBean;
import org.apache.commons.lang3.ClassUtils;
class DynaBeanBuilderSupport {
@ -48,8 +47,8 @@ class DynaBeanBuilderSupport {
private final Multimap<String, TypeTag> properties = HashMultimap.create();
private final Map<String, Object> values = new HashMap<>();
DynaBeanBuilderSupport(Class<?> destinedClass, ConvertUtilsBean convertUtilsBean,
List<String> classPrefixSearchList) {
DynaBeanBuilderSupport(
Class<?> destinedClass, ConvertUtilsBean convertUtilsBean, List<String> classPrefixSearchList) {
this.destinedClass = destinedClass;
this.convertUtilsBean = convertUtilsBean;
this.classPrefixSearchList = classPrefixSearchList;
@ -103,11 +102,12 @@ class DynaBeanBuilderSupport {
private Object createForProperty(String name) {
Optional<TypeTag> type = properties.get(name).stream().findFirst();
return type.map(t -> {
if (DynaBeanBuilderUtils.isBuilderOrCreate(t.type) || !t.hasConverter) {
return new BuilderDynaBean(t.type, convertUtilsBean, null, classPrefixSearchList);
}
return null;
}).orElse(null);
if (DynaBeanBuilderUtils.isBuilderOrCreate(t.type) || !t.hasConverter) {
return new BuilderDynaBean(t.type, convertUtilsBean, null, classPrefixSearchList);
}
return null;
})
.orElse(null);
}
boolean hasValue(String name) {
@ -157,8 +157,11 @@ class DynaBeanBuilderSupport {
void set(String name, Object value) {
if (value instanceof String && properties.get(name).stream().anyMatch(t -> t.type.isEnum())) {
TypeTag typeTag = properties.get(name).stream().filter(t -> t.type.isEnum()).findFirst().orElseThrow(
() -> new IllegalStateException("Expected enum type for " + name + ", but couldn't find it."));
TypeTag typeTag = properties.get(name).stream()
.filter(t -> t.type.isEnum())
.findFirst()
.orElseThrow(() ->
new IllegalStateException("Expected enum type for " + name + ", but couldn't find it."));
Class<? extends Enum> enumClass = (Class<? extends Enum>) typeTag.type;
values.put(name, Enum.valueOf(enumClass, value.toString()));
} else {
@ -174,9 +177,11 @@ class DynaBeanBuilderSupport {
private Object getArgument(Map.Entry<String, Object> setValue) {
Object argument = setValue.getValue();
if (argument instanceof Object[]) {
TypeTag arrayType = properties.get(setValue.getKey()).stream().filter(t -> t.type.isArray()).findFirst()
.orElseThrow(() -> new IllegalStateException(String
.format("Received Object[] for %s but can't find corresponding type", setValue.getKey())));
TypeTag arrayType = properties.get(setValue.getKey()).stream()
.filter(t -> t.type.isArray())
.findFirst()
.orElseThrow(() -> new IllegalStateException(String.format(
"Received Object[] for %s but can't find corresponding type", setValue.getKey())));
Object[] arrayValues = (Object[]) argument;
Object[] destination = (Object[]) Array.newInstance(arrayType.type.getComponentType(), arrayValues.length);
@ -212,10 +217,12 @@ class DynaBeanBuilderSupport {
for (Map.Entry<String, Object> setValue : values.entrySet()) {
Object argument = getArgument(setValue);
Method mutator = properties.get(setValue.getKey()).stream()
.filter(t -> ClassUtils.isAssignable(argument.getClass(), t.type)).findFirst()
.map(a -> a.builderMethod).orElseThrow(
() -> new IllegalStateException(String.format("Unable to find mutator for %s of type %s",
setValue.getKey(), argument.getClass().getName())));
.filter(t -> ClassUtils.isAssignable(argument.getClass(), t.type))
.findFirst()
.map(a -> a.builderMethod)
.orElseThrow(() -> new IllegalStateException(String.format(
"Unable to find mutator for %s of type %s",
setValue.getKey(), argument.getClass().getName())));
try {
source = mutator.invoke(source, argument);
} catch (IllegalAccessException | InvocationTargetException e) {
@ -236,7 +243,6 @@ class DynaBeanBuilderSupport {
} catch (IllegalAccessException | NoSuchMethodException | InvocationTargetException e) {
throw new RuntimeException(e);
}
}
Collection<String> getPropertyNames() {
@ -249,5 +255,4 @@ class DynaBeanBuilderSupport {
}
return new ArrayList<>(properties.get(name));
}
}

20
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/DynaBeanCreateSupport.java
View file

@ -30,8 +30,8 @@ class DynaBeanCreateSupport {
private final List<TypeTag> createTypes = new ArrayList<>();
private Object[] createValues = null;
DynaBeanCreateSupport(Class<?> destinedClass, ConvertUtilsBean convertUtilsBean,
List<String> classPrefixSearchList) {
DynaBeanCreateSupport(
Class<?> destinedClass, ConvertUtilsBean convertUtilsBean, List<String> classPrefixSearchList) {
this.destinedClass = destinedClass;
this.convertUtilsBean = convertUtilsBean;
this.classPrefixSearchList = classPrefixSearchList;
@ -58,8 +58,8 @@ class DynaBeanCreateSupport {
Object build() {
Method createMethod = DynaBeanBuilderUtils.getMethod(destinedClass, "create",
createTypes.stream().map(t -> t.type).toArray(i -> new Class<?>[i]));
Method createMethod = DynaBeanBuilderUtils.getMethod(
destinedClass, "create", createTypes.stream().map(t -> t.type).toArray(i -> new Class<?>[i]));
Object arguments[] = new Object[createValues.length];
for (int i = 0; i < createValues.length; ++i) {
if (createValues[i] instanceof BuilderDynaBean) {
@ -77,8 +77,8 @@ class DynaBeanCreateSupport {
return createValues[index];
} else {
if (createValues[index] == null) {
createValues[index] = new BuilderDynaBean(createTypes.get(index).type, convertUtilsBean, null,
classPrefixSearchList);
createValues[index] = new BuilderDynaBean(
createTypes.get(index).type, convertUtilsBean, null, classPrefixSearchList);
}
return createValues[index];
}
@ -89,13 +89,11 @@ class DynaBeanCreateSupport {
public void set(String name, int index, Object value) {
if (StringUtils.isEmpty(name)) {
if (index >= createValues.length) {
throw new IllegalArgumentException(
String.format("%d exceeds the maximum number of arguments (%d) for %s", index,
createValues.length, destinedClass.getName()));
throw new IllegalArgumentException(String.format(
"%d exceeds the maximum number of arguments (%d) for %s",
index, createValues.length, destinedClass.getName()));
}
createValues[index] = value;
}
}
}

13
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/FanoutConfigBean.java
View file

@ -26,21 +26,28 @@ public class FanoutConfigBean implements RetrievalConfigBuilder {
@ConfigurationSettable(configurationClass = FanOutConfig.class)
private int maxDescribeStreamSummaryRetries;
@ConfigurationSettable(configurationClass = FanOutConfig.class)
private String consumerArn;
@ConfigurationSettable(configurationClass = FanOutConfig.class)
private String consumerName;
@ConfigurationSettable(configurationClass = FanOutConfig.class)
private int maxDescribeStreamConsumerRetries;
@ConfigurationSettable(configurationClass = FanOutConfig.class)
private int registerStreamConsumerRetries;
@ConfigurationSettable(configurationClass = FanOutConfig.class)
private long retryBackoffMillis;
@Override
public FanOutConfig build(KinesisAsyncClient kinesisAsyncClient, MultiLangDaemonConfiguration parent) {
return ConfigurationSettableUtils.resolveFields(this, new FanOutConfig(kinesisAsyncClient).applicationName(parent.getApplicationName())
.streamName(parent.getStreamName()));
return ConfigurationSettableUtils.resolveFields(
this,
new FanOutConfig(kinesisAsyncClient)
.applicationName(parent.getApplicationName())
.streamName(parent.getStreamName()));
}
}

3
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/IntegerPropertyValueDecoder.java
View file

@ -25,8 +25,7 @@ class IntegerPropertyValueDecoder implements IPropertyValueDecoder<Integer> {
/**
* Constructor.
*/
IntegerPropertyValueDecoder() {
}
IntegerPropertyValueDecoder() {}
/**
* @param value property value as String

17
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/KinesisClientLibConfigurator.java
View file

@ -19,10 +19,9 @@ import java.io.InputStream;
import java.lang.reflect.InvocationTargetException;
import java.util.Properties;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.beanutils.BeanUtilsBean;
import org.apache.commons.beanutils.ConvertUtilsBean;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.Validate;
import software.amazon.awssdk.arns.Arn;
import software.amazon.kinesis.common.StreamIdentifier;
@ -71,16 +70,21 @@ public class KinesisClientLibConfigurator {
Validate.notBlank(configuration.getApplicationName(), "Application name is required");
if (configuration.getStreamArn() != null && !configuration.getStreamArn().trim().isEmpty()) {
if (configuration.getStreamArn() != null
&& !configuration.getStreamArn().trim().isEmpty()) {
final Arn streamArnObj = Arn.fromString(configuration.getStreamArn());
StreamIdentifier.validateArn(streamArnObj);
//Parse out the stream Name from the Arn (and/or override existing value for Stream Name)
// Parse out the stream Name from the Arn (and/or override existing value for Stream Name)
final String streamNameFromArn = streamArnObj.resource().resource();
configuration.setStreamName(streamNameFromArn);
}
Validate.notBlank(configuration.getStreamName(), "Stream name or Stream Arn is required. Stream Arn takes precedence if both are passed in.");
Validate.isTrue(configuration.getKinesisCredentialsProvider().isDirty(), "A basic set of AWS credentials must be provided");
Validate.notBlank(
configuration.getStreamName(),
"Stream name or Stream Arn is required. Stream Arn takes precedence if both are passed in.");
Validate.isTrue(
configuration.getKinesisCredentialsProvider().isDirty(),
"A basic set of AWS credentials must be provided");
return configuration;
}
@ -106,5 +110,4 @@ public class KinesisClientLibConfigurator {
}
return getConfiguration(properties);
}
}

195
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/MultiLangDaemonConfiguration.java
View file

@ -27,17 +27,16 @@ import java.util.Set;
import java.util.UUID;
import java.util.function.Function;
import org.apache.commons.beanutils.BeanUtilsBean;
import org.apache.commons.beanutils.ConvertUtilsBean;
import org.apache.commons.beanutils.Converter;
import org.apache.commons.beanutils.converters.ArrayConverter;
import org.apache.commons.beanutils.converters.StringConverter;
import lombok.Data;
import lombok.Getter;
import lombok.Setter;
import lombok.experimental.Delegate;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.beanutils.BeanUtilsBean;
import org.apache.commons.beanutils.ConvertUtilsBean;
import org.apache.commons.beanutils.Converter;
import org.apache.commons.beanutils.converters.ArrayConverter;
import org.apache.commons.beanutils.converters.StringConverter;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient;
@ -74,7 +73,6 @@ public class MultiLangDaemonConfiguration {
private String streamName;
private String streamArn;
@ConfigurationSettable(configurationClass = ConfigsBuilder.class)
private String tableName;
@ -86,22 +84,31 @@ public class MultiLangDaemonConfiguration {
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private long failoverTimeMillis;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private Boolean enablePriorityLeaseAssignment;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private long shardSyncIntervalMillis;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private boolean cleanupLeasesUponShardCompletion;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private boolean ignoreUnexpectedChildShards;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private int maxLeasesForWorker;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private int maxLeasesToStealAtOneTime;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private int initialLeaseTableReadCapacity;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private int initialLeaseTableWriteCapacity;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class, methodName = "initialPositionInStream")
@ConfigurationSettable(configurationClass = RetrievalConfig.class)
private InitialPositionInStreamExtended initialPositionInStreamExtended;
@ -114,14 +121,16 @@ public class MultiLangDaemonConfiguration {
}
public void setInitialPositionInStream(InitialPositionInStream initialPositionInStream) {
this.initialPositionInStreamExtended = InitialPositionInStreamExtended
.newInitialPosition(initialPositionInStream);
this.initialPositionInStreamExtended =
InitialPositionInStreamExtended.newInitialPosition(initialPositionInStream);
}
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private int maxLeaseRenewalThreads;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private long listShardsBackoffTimeInMillis;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private int maxListShardsRetryAttempts;
@ -131,10 +140,13 @@ public class MultiLangDaemonConfiguration {
@ConfigurationSettable(configurationClass = CoordinatorConfig.class)
private long parentShardPollIntervalMillis;
@ConfigurationSettable(configurationClass = CoordinatorConfig.class)
private ShardPrioritization shardPrioritization;
@ConfigurationSettable(configurationClass = CoordinatorConfig.class)
private boolean skipShardSyncAtWorkerInitializationIfLeasesExist;
@ConfigurationSettable(configurationClass = CoordinatorConfig.class)
private long schedulerInitializationBackoffTimeMillis;
@ -143,12 +155,16 @@ public class MultiLangDaemonConfiguration {
@ConfigurationSettable(configurationClass = MetricsConfig.class)
private long metricsBufferTimeMillis;
@ConfigurationSettable(configurationClass = MetricsConfig.class)
private int metricsMaxQueueSize;
@ConfigurationSettable(configurationClass = MetricsConfig.class)
private MetricsLevel metricsLevel;
@ConfigurationSettable(configurationClass = LifecycleConfig.class, convertToOptional = true)
private Long logWarningForTaskAfterMillis;
@ConfigurationSettable(configurationClass = MetricsConfig.class)
private Set<String> metricsEnabledDimensions;
@ -163,6 +179,7 @@ public class MultiLangDaemonConfiguration {
private RetrievalMode retrievalMode = RetrievalMode.DEFAULT;
private final FanoutConfigBean fanoutConfig = new FanoutConfigBean();
@Delegate(types = PollingConfigBean.PollingConfigBeanDelegate.class)
private final PollingConfigBean pollingConfig = new PollingConfigBean();
@ -200,61 +217,75 @@ public class MultiLangDaemonConfiguration {
this.utilsBean = utilsBean;
this.convertUtilsBean = convertUtilsBean;
convertUtilsBean.register(new Converter() {
@Override
public <T> T convert(Class<T> type, Object value) {
Date date = new Date(Long.parseLong(value.toString()) * 1000L);
return type.cast(InitialPositionInStreamExtended.newInitialPositionAtTimestamp(date));
}
}, InitialPositionInStreamExtended.class);
convertUtilsBean.register(
new Converter() {
@Override
public <T> T convert(Class<T> type, Object value) {
Date date = new Date(Long.parseLong(value.toString()) * 1000L);
return type.cast(InitialPositionInStreamExtended.newInitialPositionAtTimestamp(date));
}
},
InitialPositionInStreamExtended.class);
convertUtilsBean.register(new Converter() {
@Override
public <T> T convert(Class<T> type, Object value) {
return type.cast(MetricsLevel.valueOf(value.toString().toUpperCase()));
}
}, MetricsLevel.class);
convertUtilsBean.register(
new Converter() {
@Override
public <T> T convert(Class<T> type, Object value) {
return type.cast(MetricsLevel.valueOf(value.toString().toUpperCase()));
}
},
MetricsLevel.class);
convertUtilsBean.register(new Converter() {
@Override
public <T> T convert(Class<T> type, Object value) {
return type.cast(InitialPositionInStream.valueOf(value.toString().toUpperCase()));
}
}, InitialPositionInStream.class);
convertUtilsBean.register(
new Converter() {
@Override
public <T> T convert(Class<T> type, Object value) {
return type.cast(
InitialPositionInStream.valueOf(value.toString().toUpperCase()));
}
},
InitialPositionInStream.class);
convertUtilsBean.register(new Converter() {
@Override
public <T> T convert(Class<T> type, Object value) {
return type.cast(URI.create(value.toString()));
}
}, URI.class);
convertUtilsBean.register(
new Converter() {
@Override
public <T> T convert(Class<T> type, Object value) {
return type.cast(URI.create(value.toString()));
}
},
URI.class);
convertUtilsBean.register(new Converter() {
@Override
public <T> T convert(Class<T> type, Object value) {
return type.cast(RetrievalMode.from(value.toString()));
}
}, RetrievalMode.class);
convertUtilsBean.register(
new Converter() {
@Override
public <T> T convert(Class<T> type, Object value) {
return type.cast(RetrievalMode.from(value.toString()));
}
},
RetrievalMode.class);
convertUtilsBean.register(new Converter() {
@Override
public <T> T convert(final Class<T> type, final Object value) {
return type.cast(Region.of(value.toString()));
}
}, Region.class);
convertUtilsBean.register(
new Converter() {
@Override
public <T> T convert(final Class<T> type, final Object value) {
return type.cast(Region.of(value.toString()));
}
},
Region.class);
ArrayConverter arrayConverter = new ArrayConverter(String[].class, new StringConverter());
arrayConverter.setDelimiter(',');
convertUtilsBean.register(arrayConverter, String[].class);
AWSCredentialsProviderPropertyValueDecoder oldCredentialsDecoder = new AWSCredentialsProviderPropertyValueDecoder();
AWSCredentialsProviderPropertyValueDecoder oldCredentialsDecoder =
new AWSCredentialsProviderPropertyValueDecoder();
Function<String, ?> converter = s -> new V2CredentialWrapper(oldCredentialsDecoder.decodeValue(s));
this.kinesisCredentialsProvider = new BuilderDynaBean(AwsCredentialsProvider.class, convertUtilsBean,
converter, CREDENTIALS_DEFAULT_SEARCH_PATH);
this.dynamoDBCredentialsProvider = new BuilderDynaBean(AwsCredentialsProvider.class, convertUtilsBean,
converter, CREDENTIALS_DEFAULT_SEARCH_PATH);
this.cloudWatchCredentialsProvider = new BuilderDynaBean(AwsCredentialsProvider.class, convertUtilsBean,
converter, CREDENTIALS_DEFAULT_SEARCH_PATH);
this.kinesisCredentialsProvider = new BuilderDynaBean(
AwsCredentialsProvider.class, convertUtilsBean, converter, CREDENTIALS_DEFAULT_SEARCH_PATH);
this.dynamoDBCredentialsProvider = new BuilderDynaBean(
AwsCredentialsProvider.class, convertUtilsBean, converter, CREDENTIALS_DEFAULT_SEARCH_PATH);
this.cloudWatchCredentialsProvider = new BuilderDynaBean(
AwsCredentialsProvider.class, convertUtilsBean, converter, CREDENTIALS_DEFAULT_SEARCH_PATH);
this.kinesisClient = new BuilderDynaBean(KinesisAsyncClient.class, convertUtilsBean);
this.dynamoDbClient = new BuilderDynaBean(DynamoDbAsyncClient.class, convertUtilsBean);
@ -300,8 +331,8 @@ public class MultiLangDaemonConfiguration {
return credsBuilder.build(AwsCredentialsProvider.class);
}
private void updateCredentials(BuilderDynaBean toUpdate, AwsCredentialsProvider primary,
AwsCredentialsProvider secondary) {
private void updateCredentials(
BuilderDynaBean toUpdate, AwsCredentialsProvider primary, AwsCredentialsProvider secondary) {
if (toUpdate.hasValue("credentialsProvider")) {
return;
@ -329,8 +360,8 @@ public class MultiLangDaemonConfiguration {
}
private void handleRetrievalConfig(RetrievalConfig retrievalConfig, ConfigsBuilder configsBuilder) {
retrievalConfig
.retrievalSpecificConfig(retrievalMode.builder(this).build(configsBuilder.kinesisClient(), this));
retrievalConfig.retrievalSpecificConfig(
retrievalMode.builder(this).build(configsBuilder.kinesisClient(), this));
}
private Object adjustKinesisHttpConfiguration(Object builderObj) {
@ -353,8 +384,14 @@ public class MultiLangDaemonConfiguration {
final RetrievalConfig retrievalConfig;
public Scheduler build() {
return new Scheduler(checkpointConfig, coordinatorConfig, leaseManagementConfig, lifecycleConfig,
metricsConfig, processorConfig, retrievalConfig);
return new Scheduler(
checkpointConfig,
coordinatorConfig,
leaseManagementConfig,
lifecycleConfig,
metricsConfig,
processorConfig,
retrievalConfig);
}
}
@ -367,19 +404,25 @@ public class MultiLangDaemonConfiguration {
updateCredentials(dynamoDbClient, dynamoDbCreds, kinesisCreds);
updateCredentials(cloudWatchClient, cloudwatchCreds, kinesisCreds);
KinesisAsyncClient kinesisAsyncClient = kinesisClient.build(KinesisAsyncClient.class,
this::adjustKinesisHttpConfiguration);
KinesisAsyncClient kinesisAsyncClient =
kinesisClient.build(KinesisAsyncClient.class, this::adjustKinesisHttpConfiguration);
DynamoDbAsyncClient dynamoDbAsyncClient = dynamoDbClient.build(DynamoDbAsyncClient.class);
CloudWatchAsyncClient cloudWatchAsyncClient = cloudWatchClient.build(CloudWatchAsyncClient.class);
ConfigsBuilder configsBuilder = new ConfigsBuilder(streamName, applicationName, kinesisAsyncClient,
dynamoDbAsyncClient, cloudWatchAsyncClient, workerIdentifier, shardRecordProcessorFactory);
ConfigsBuilder configsBuilder = new ConfigsBuilder(
streamName,
applicationName,
kinesisAsyncClient,
dynamoDbAsyncClient,
cloudWatchAsyncClient,
workerIdentifier,
shardRecordProcessorFactory);
Map<Class<?>, Object> configObjects = new HashMap<>();
addConfigObjects(configObjects, configsBuilder);
resolveFields(configObjects, Collections.singleton(ConfigsBuilder.class),
Collections.singleton(PollingConfig.class));
resolveFields(
configObjects, Collections.singleton(ConfigsBuilder.class), Collections.singleton(PollingConfig.class));
CoordinatorConfig coordinatorConfig = configsBuilder.coordinatorConfig();
CheckpointConfig checkpointConfig = configsBuilder.checkpointConfig();
@ -389,19 +432,31 @@ public class MultiLangDaemonConfiguration {
ProcessorConfig processorConfig = configsBuilder.processorConfig();
RetrievalConfig retrievalConfig = configsBuilder.retrievalConfig();
addConfigObjects(configObjects, coordinatorConfig, checkpointConfig, leaseManagementConfig, lifecycleConfig,
metricsConfig, processorConfig, retrievalConfig);
addConfigObjects(
configObjects,
coordinatorConfig,
checkpointConfig,
leaseManagementConfig,
lifecycleConfig,
metricsConfig,
processorConfig,
retrievalConfig);
handleRetrievalConfig(retrievalConfig, configsBuilder);
resolveFields(configObjects, null, new HashSet<>(Arrays.asList(ConfigsBuilder.class, PollingConfig.class)));
return new ResolvedConfiguration(coordinatorConfig, checkpointConfig, leaseManagementConfig, lifecycleConfig,
metricsConfig, processorConfig, retrievalConfig);
return new ResolvedConfiguration(
coordinatorConfig,
checkpointConfig,
leaseManagementConfig,
lifecycleConfig,
metricsConfig,
processorConfig,
retrievalConfig);
}
public Scheduler build(ShardRecordProcessorFactory shardRecordProcessorFactory) {
return resolvedConfiguration(shardRecordProcessorFactory).build();
}
}

16
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/PollingConfigBean.java
View file

@ -30,34 +30,44 @@ public class PollingConfigBean implements RetrievalConfigBuilder {
interface PollingConfigBeanDelegate {
Integer getRetryGetRecordsInSeconds();
void setRetryGetRecordsInSeconds(Integer value);
Integer getMaxGetRecordsThreadPool();
void setMaxGetRecordsThreadPool(Integer value);
long getIdleTimeBetweenReadsInMillis();
void setIdleTimeBetweenReadsInMillis(long value);
int getMaxRecords();
void setMaxRecords(int value);
}
@ConfigurationSettable(configurationClass = PollingConfig.class, convertToOptional = true)
private Integer retryGetRecordsInSeconds;
@ConfigurationSettable(configurationClass = PollingConfig.class, convertToOptional = true)
private Integer maxGetRecordsThreadPool;
@ConfigurationSettable(configurationClass = PollingConfig.class)
private long idleTimeBetweenReadsInMillis;
@ConfigurationSettable(configurationClass = PollingConfig.class)
private int maxRecords;
public boolean anyPropertiesSet() {
return retryGetRecordsInSeconds != null || maxGetRecordsThreadPool != null || idleTimeBetweenReadsInMillis != 0 || maxRecords != 0;
return retryGetRecordsInSeconds != null
|| maxGetRecordsThreadPool != null
|| idleTimeBetweenReadsInMillis != 0
|| maxRecords != 0;
}
@Override
public PollingConfig build(KinesisAsyncClient kinesisAsyncClient, MultiLangDaemonConfiguration parent) {
return ConfigurationSettableUtils.resolveFields(this, new PollingConfig(parent.getStreamName(), kinesisAsyncClient));
return ConfigurationSettableUtils.resolveFields(
this, new PollingConfig(parent.getStreamName(), kinesisAsyncClient));
}
}

8
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/RetrievalMode.java
View file

@ -19,14 +19,14 @@ import java.util.Arrays;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.commons.lang3.Validate;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.Validate;
@Slf4j
public enum RetrievalMode {
FANOUT(MultiLangDaemonConfiguration::getFanoutConfig), POLLING(
MultiLangDaemonConfiguration::getPollingConfig), DEFAULT(RetrievalMode::decideForDefault);
FANOUT(MultiLangDaemonConfiguration::getFanoutConfig),
POLLING(MultiLangDaemonConfiguration::getPollingConfig),
DEFAULT(RetrievalMode::decideForDefault);
private final Function<MultiLangDaemonConfiguration, RetrievalConfigBuilder> builderFor;

5
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/TypeTag.java
View file

@ -15,14 +15,13 @@
package software.amazon.kinesis.multilang.config;
import lombok.Data;
import java.lang.reflect.Method;
import lombok.Data;
@Data
class TypeTag {
final Class<?> type;
final boolean hasConverter;
final Method builderMethod;
}

4
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/credentials/V2CredentialWrapper.java
View file

@ -32,7 +32,9 @@ public class V2CredentialWrapper implements AwsCredentialsProvider {
public AwsCredentials resolveCredentials() {
AWSCredentials current = oldCredentialsProvider.getCredentials();
if (current instanceof AWSSessionCredentials) {
return AwsSessionCredentials.create(current.getAWSAccessKeyId(), current.getAWSSecretKey(),
return AwsSessionCredentials.create(
current.getAWSAccessKeyId(),
current.getAWSSecretKey(),
((AWSSessionCredentials) current).getSessionToken());
}
return new AwsCredentials() {

2
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/CheckpointMessage.java
View file

@ -36,6 +36,7 @@ public class CheckpointMessage extends Message {
* The checkpoint this message is about.
*/
private String sequenceNumber;
private Long subSequenceNumber;
/**
@ -61,5 +62,4 @@ public class CheckpointMessage extends Message {
this.setError(throwable.getClass().getSimpleName());
}
}
}

8
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/InitializeMessage.java
View file

@ -33,14 +33,14 @@ public class InitializeMessage extends Message {
* The shard id that this processor is getting initialized for.
*/
private String shardId;
private String sequenceNumber;
private Long subSequenceNumber;
/**
* Default constructor.
*/
public InitializeMessage() {
}
public InitializeMessage() {}
/**
* Convenience constructor.
@ -51,11 +51,11 @@ public class InitializeMessage extends Message {
this.shardId = initializationInput.shardId();
if (initializationInput.extendedSequenceNumber() != null) {
this.sequenceNumber = initializationInput.extendedSequenceNumber().sequenceNumber();
this.subSequenceNumber = initializationInput.extendedSequenceNumber().subSequenceNumber();
this.subSequenceNumber =
initializationInput.extendedSequenceNumber().subSequenceNumber();
} else {
this.sequenceNumber = null;
this.subSequenceNumber = null;
}
}
}

8
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/JsonFriendlyRecord.java
View file

@ -15,7 +15,6 @@
package software.amazon.kinesis.multilang.messages;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.EqualsAndHashCode;
import lombok.Getter;
@ -54,10 +53,11 @@ public class JsonFriendlyRecord {
data = new byte[record.data().limit()];
record.data().get(data);
}
Long approximateArrival = record.approximateArrivalTimestamp() == null ? null
Long approximateArrival = record.approximateArrivalTimestamp() == null
? null
: record.approximateArrivalTimestamp().toEpochMilli();
return new JsonFriendlyRecord(data, record.partitionKey(), record.sequenceNumber(),
approximateArrival, record.subSequenceNumber());
return new JsonFriendlyRecord(
data, record.partitionKey(), record.sequenceNumber(), approximateArrival, record.subSequenceNumber());
}
@JsonProperty

1
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/LeaseLostMessage.java
View file

@ -21,5 +21,4 @@ package software.amazon.kinesis.multilang.messages;
public class LeaseLostMessage extends Message {
public static final String ACTION = "leaseLost";
}

19
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/Message.java
View file

@ -24,14 +24,14 @@ import com.fasterxml.jackson.databind.ObjectMapper;
*/
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "action")
@JsonSubTypes({
@Type(value = CheckpointMessage.class, name = CheckpointMessage.ACTION),
@Type(value = InitializeMessage.class, name = InitializeMessage.ACTION),
@Type(value = ProcessRecordsMessage.class, name = ProcessRecordsMessage.ACTION),
@Type(value = ShutdownMessage.class, name = ShutdownMessage.ACTION),
@Type(value = StatusMessage.class, name = StatusMessage.ACTION),
@Type(value = ShutdownRequestedMessage.class, name = ShutdownRequestedMessage.ACTION),
@Type(value = LeaseLostMessage.class, name = LeaseLostMessage.ACTION),
@Type(value = ShardEndedMessage.class, name = ShardEndedMessage.ACTION),
@Type(value = CheckpointMessage.class, name = CheckpointMessage.ACTION),
@Type(value = InitializeMessage.class, name = InitializeMessage.ACTION),
@Type(value = ProcessRecordsMessage.class, name = ProcessRecordsMessage.ACTION),
@Type(value = ShutdownMessage.class, name = ShutdownMessage.ACTION),
@Type(value = StatusMessage.class, name = StatusMessage.ACTION),
@Type(value = ShutdownRequestedMessage.class, name = ShutdownRequestedMessage.ACTION),
@Type(value = LeaseLostMessage.class, name = LeaseLostMessage.ACTION),
@Type(value = ShardEndedMessage.class, name = ShardEndedMessage.ACTION),
})
public abstract class Message {
@ -40,8 +40,7 @@ public abstract class Message {
/**
* Default constructor.
*/
public Message() {
}
public Message() {}
/**
*

4
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/ProcessRecordsMessage.java
View file

@ -37,13 +37,13 @@ public class ProcessRecordsMessage extends Message {
* The records that the client's process needs to handle.
*/
private List<JsonFriendlyRecord> records;
private Long millisBehindLatest;
/**
* Default constructor.
*/
public ProcessRecordsMessage() {
}
public ProcessRecordsMessage() {}
/**
* Convenience constructor.

1
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/package-info.java
View file

@ -145,4 +145,3 @@
*
*/
package software.amazon.kinesis.multilang;

24
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/Matchers.java
View file

@ -14,15 +14,14 @@
*/
package software.amazon.kinesis.multilang;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.nullValue;
import org.hamcrest.Description;
import org.hamcrest.Matcher;
import org.hamcrest.TypeSafeDiagnosingMatcher;
import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber;
import software.amazon.kinesis.lifecycle.events.InitializationInput;
import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.nullValue;
public class Matchers {
@ -58,8 +57,12 @@ public class Matchers {
@Override
public void describeTo(Description description) {
description.appendText("An InitializationInput matching: { shardId: ").appendDescriptionOf(shardIdMatcher)
.appendText(", sequenceNumber: ").appendDescriptionOf(sequenceNumberMatcher).appendText(" }");
description
.appendText("An InitializationInput matching: { shardId: ")
.appendDescriptionOf(shardIdMatcher)
.appendText(", sequenceNumber: ")
.appendDescriptionOf(sequenceNumberMatcher)
.appendText(" }");
}
}
@ -98,10 +101,11 @@ public class Matchers {
@Override
public void describeTo(Description description) {
description.appendText("An ExtendedSequenceNumber matching: { sequenceNumber: ")
.appendDescriptionOf(sequenceNumberMatcher).appendText(", subSequenceNumber: ")
description
.appendText("An ExtendedSequenceNumber matching: { sequenceNumber: ")
.appendDescriptionOf(sequenceNumberMatcher)
.appendText(", subSequenceNumber: ")
.appendDescriptionOf(subSequenceNumberMatcher);
}
}
}

47
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MessageReaderTest.java
View file

@ -22,15 +22,14 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import software.amazon.kinesis.multilang.messages.Message;
import software.amazon.kinesis.multilang.messages.StatusMessage;
import com.fasterxml.jackson.databind.ObjectMapper;
public class MessageReaderTest {
@ -75,8 +74,8 @@ public class MessageReaderTest {
@Test
public void runLoopGoodInputTest() {
String[] sequenceNumbers = new String[] { "123", "456", "789" };
String[] responseFors = new String[] { "initialize", "processRecords", "processRecords", "shutdown" };
String[] sequenceNumbers = new String[] {"123", "456", "789"};
String[] responseFors = new String[] {"initialize", "processRecords", "processRecords", "shutdown"};
InputStream stream = buildInputStreamOfGoodInput(sequenceNumbers, responseFors);
MessageReader reader =
new MessageReader().initialize(stream, SHARD_ID, new ObjectMapper(), Executors.newCachedThreadPool());
@ -85,7 +84,9 @@ public class MessageReaderTest {
try {
Message message = reader.getNextMessageFromSTDOUT().get();
if (message instanceof StatusMessage) {
Assert.assertEquals("The status message's responseFor field should have been correct", responseFor,
Assert.assertEquals(
"The status message's responseFor field should have been correct",
responseFor,
((StatusMessage) message).getResponseFor());
}
} catch (InterruptedException | ExecutionException e) {
@ -96,8 +97,8 @@ public class MessageReaderTest {
@Test
public void drainInputTest() throws InterruptedException, ExecutionException {
String[] sequenceNumbers = new String[] { "123", "456", "789" };
String[] responseFors = new String[] { "initialize", "processRecords", "processRecords", "shutdown" };
String[] sequenceNumbers = new String[] {"123", "456", "789"};
String[] responseFors = new String[] {"initialize", "processRecords", "processRecords", "shutdown"};
InputStream stream = buildInputStreamOfGoodInput(sequenceNumbers, responseFors);
MessageReader reader =
@ -116,25 +117,26 @@ public class MessageReaderTest {
BufferedReader bufferReader = Mockito.mock(BufferedReader.class);
try {
Mockito.doAnswer(new Answer() {
private boolean returnedOnce = false;
private boolean returnedOnce = false;
@Override
public Object answer(InvocationOnMock invocation) throws Throwable {
if (returnedOnce) {
return "{\"action\":\"status\",\"responseFor\":\"processRecords\"}";
} else {
returnedOnce = true;
return "{\"action\":\"shutdown\",\"reason\":\"ZOMBIE\"}";
}
}
}).when(bufferReader).readLine();
@Override
public Object answer(InvocationOnMock invocation) throws Throwable {
if (returnedOnce) {
return "{\"action\":\"status\",\"responseFor\":\"processRecords\"}";
} else {
returnedOnce = true;
return "{\"action\":\"shutdown\",\"reason\":\"ZOMBIE\"}";
}
}
})
.when(bufferReader)
.readLine();
} catch (IOException e) {
Assert.fail("There shouldn't be an exception while setting up this mock.");
}
MessageReader reader =
new MessageReader().initialize(bufferReader, SHARD_ID, new ObjectMapper(),
Executors.newCachedThreadPool());
MessageReader reader = new MessageReader()
.initialize(bufferReader, SHARD_ID, new ObjectMapper(), Executors.newCachedThreadPool());
try {
reader.getNextMessageFromSTDOUT().get();
@ -165,7 +167,8 @@ public class MessageReaderTest {
readTask.get();
Assert.fail("The reading task should have failed due to an IOException.");
} catch (InterruptedException e) {
Assert.fail("The reading task should not have been interrupted. It should have failed due to an IOException.");
Assert.fail(
"The reading task should not have been interrupted. It should have failed due to an IOException.");
} catch (ExecutionException e) {
// Yay!!
}

53
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MessageWriterTest.java
View file

@ -23,21 +23,19 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.mockito.Mockito;
import software.amazon.kinesis.lifecycle.events.InitializationInput;
import software.amazon.kinesis.lifecycle.events.LeaseLostInput;
import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput;
import software.amazon.kinesis.lifecycle.events.ShardEndedInput;
import software.amazon.kinesis.multilang.messages.Message;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import software.amazon.kinesis.lifecycle.events.InitializationInput;
import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput;
import software.amazon.kinesis.retrieval.KinesisClientRecord;
import static org.mockito.Mockito.verify;
@ -65,8 +63,7 @@ public class MessageWriterTest {
public void writeCheckpointMessageNoErrorTest() throws IOException, InterruptedException, ExecutionException {
Future<Boolean> future = this.messageWriter.writeCheckpointMessageWithError("1234", 0L, null);
future.get();
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(),
Mockito.anyInt());
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt());
verify(this.stream, Mockito.atLeastOnce()).flush();
}
@ -74,42 +71,43 @@ public class MessageWriterTest {
public void writeCheckpointMessageWithErrorTest() throws IOException, InterruptedException, ExecutionException {
Future<Boolean> future = this.messageWriter.writeCheckpointMessageWithError("1234", 0L, new Throwable());
future.get();
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(),
Mockito.anyInt());
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt());
verify(this.stream, Mockito.atLeastOnce()).flush();
}
@Test
public void writeInitializeMessageTest() throws IOException, InterruptedException, ExecutionException {
Future<Boolean> future = this.messageWriter.writeInitializeMessage(InitializationInput.builder().shardId(SHARD_ID).build());
Future<Boolean> future = this.messageWriter.writeInitializeMessage(
InitializationInput.builder().shardId(SHARD_ID).build());
future.get();
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(),
Mockito.anyInt());
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt());
verify(this.stream, Mockito.atLeastOnce()).flush();
}
@Test
public void writeProcessRecordsMessageTest() throws IOException, InterruptedException, ExecutionException {
List<KinesisClientRecord> records = Arrays.asList(
KinesisClientRecord.builder().data(ByteBuffer.wrap("kitten".getBytes())).partitionKey("some cats")
.sequenceNumber("357234807854789057805").build(),
KinesisClientRecord.builder().build()
);
Future<Boolean> future = this.messageWriter.writeProcessRecordsMessage(ProcessRecordsInput.builder().records(records).build());
KinesisClientRecord.builder()
.data(ByteBuffer.wrap("kitten".getBytes()))
.partitionKey("some cats")
.sequenceNumber("357234807854789057805")
.build(),
KinesisClientRecord.builder().build());
Future<Boolean> future = this.messageWriter.writeProcessRecordsMessage(
ProcessRecordsInput.builder().records(records).build());
future.get();
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(),
Mockito.anyInt());
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt());
verify(this.stream, Mockito.atLeastOnce()).flush();
}
@Test
public void writeShutdownMessageTest() throws IOException, InterruptedException, ExecutionException {
Future<Boolean> future = this.messageWriter.writeShardEndedMessage(ShardEndedInput.builder().build());
Future<Boolean> future = this.messageWriter.writeShardEndedMessage(
ShardEndedInput.builder().build());
future.get();
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(),
Mockito.anyInt());
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt());
verify(this.stream, Mockito.atLeastOnce()).flush();
}
@ -118,15 +116,15 @@ public class MessageWriterTest {
Future<Boolean> future = this.messageWriter.writeShutdownRequestedMessage();
future.get();
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(),
Mockito.anyInt());
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt());
verify(this.stream, Mockito.atLeastOnce()).flush();
}
@Test
public void streamIOExceptionTest() throws IOException, InterruptedException, ExecutionException {
Mockito.doThrow(IOException.class).when(stream).flush();
Future<Boolean> initializeTask = this.messageWriter.writeInitializeMessage(InitializationInput.builder().shardId(SHARD_ID).build());
Future<Boolean> initializeTask = this.messageWriter.writeInitializeMessage(
InitializationInput.builder().shardId(SHARD_ID).build());
Boolean result = initializeTask.get();
Assert.assertNotNull(result);
Assert.assertFalse(result);
@ -152,7 +150,8 @@ public class MessageWriterTest {
Assert.assertFalse(this.messageWriter.isOpen());
try {
// Any message should fail
this.messageWriter.writeInitializeMessage(InitializationInput.builder().shardId(SHARD_ID).build());
this.messageWriter.writeInitializeMessage(
InitializationInput.builder().shardId(SHARD_ID).build());
Assert.fail("MessageWriter should be closed and unable to write.");
} catch (IllegalStateException e) {
// This should happen.

28
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MultiLangDaemonConfigTest.java
View file

@ -14,26 +14,25 @@
*/
package software.amazon.kinesis.multilang;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.mockito.Mockito.when;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import software.amazon.awssdk.regions.Region;
import junit.framework.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.runners.MockitoJUnitRunner;
import junit.framework.Assert;
import software.amazon.awssdk.auth.credentials.AwsCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.regions.Region;
import software.amazon.kinesis.multilang.config.KinesisClientLibConfigurator;
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
public class MultiLangDaemonConfigTest {
private static final String FILENAME = "multilang.properties";
@ -49,6 +48,7 @@ public class MultiLangDaemonConfigTest {
@Mock
private AwsCredentialsProvider credentialsProvider;
@Mock
private AwsCredentials creds;
@ -62,14 +62,13 @@ public class MultiLangDaemonConfigTest {
* @throws IOException
*/
public void setup(String streamName, String streamArn) throws IOException {
String properties = String.format("executableName = %s\n"
String properties = String.format(
"executableName = %s\n"
+ "applicationName = %s\n"
+ "AWSCredentialsProvider = DefaultAWSCredentialsProviderChain\n"
+ "processingLanguage = malbolge\n"
+ "regionName = %s\n",
EXE,
APPLICATION_NAME,
"us-east-1");
EXE, APPLICATION_NAME, "us-east-1");
if (streamName != null) {
properties += String.format("streamName = %s\n", streamName);
@ -79,7 +78,8 @@ public class MultiLangDaemonConfigTest {
}
classLoader = Mockito.mock(ClassLoader.class);
Mockito.doReturn(new ByteArrayInputStream(properties.getBytes())).when(classLoader)
Mockito.doReturn(new ByteArrayInputStream(properties.getBytes()))
.when(classLoader)
.getResourceAsStream(FILENAME);
when(credentialsProvider.resolveCredentials()).thenReturn(creds);
@ -185,7 +185,8 @@ public class MultiLangDaemonConfigTest {
+ "AWSCredentialsProvider = DefaultAWSCredentialsProviderChain\n" + "processingLanguage = malbolge";
ClassLoader classLoader = Mockito.mock(ClassLoader.class);
Mockito.doReturn(new ByteArrayInputStream(propertiesNoExecutableName.getBytes())).when(classLoader)
Mockito.doReturn(new ByteArrayInputStream(propertiesNoExecutableName.getBytes()))
.when(classLoader)
.getResourceAsStream(FILENAME);
try {
@ -207,5 +208,4 @@ public class MultiLangDaemonConfigTest {
public void testActualPropertiesFile() throws Exception {
new MultiLangDaemonConfig(FILENAME);
}
}

58
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MultiLangDaemonTest.java
View file

@ -14,6 +14,28 @@
*/
package software.amazon.kinesis.multilang;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.joran.JoranConfigurator;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
import org.slf4j.LoggerFactory;
import software.amazon.kinesis.coordinator.Scheduler;
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.equalTo;
@ -27,46 +49,29 @@ import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
import org.slf4j.LoggerFactory;
import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.joran.JoranConfigurator;
import software.amazon.kinesis.coordinator.Scheduler;
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
@RunWith(MockitoJUnitRunner.class)
public class MultiLangDaemonTest {
@Mock
private Scheduler scheduler;
@Mock
private MultiLangDaemonConfig config;
@Mock
private ExecutorService executorService;
@Mock
private Future<Integer> futureInteger;
@Mock
private MultiLangDaemonConfiguration multiLangDaemonConfiguration;
@Mock
private Runtime runtime;
@Rule
public ExpectedException expectedException = ExpectedException.none();
@Rule
public final TemporaryFolder temporaryFolder = new TemporaryFolder();
@ -86,7 +91,7 @@ public class MultiLangDaemonTest {
public void testSuccessfulNoOptionsJCommanderBuild() {
String testPropertiesFile = "/test/properties/file";
MultiLangDaemon.MultiLangDaemonArguments arguments = new MultiLangDaemon.MultiLangDaemonArguments();
daemon.buildJCommanderAndParseArgs(arguments, new String[] { testPropertiesFile });
daemon.buildJCommanderAndParseArgs(arguments, new String[] {testPropertiesFile});
assertThat(arguments.propertiesFile, nullValue());
assertThat(arguments.logConfiguration, nullValue());
@ -98,7 +103,7 @@ public class MultiLangDaemonTest {
public void testSuccessfulOptionsJCommanderBuild() {
String propertiesOption = "/test/properties/file/option";
String propertiesFileArgs = "/test/properties/args";
String[] args = new String[] { "-p", propertiesOption, propertiesFileArgs };
String[] args = new String[] {"-p", propertiesOption, propertiesFileArgs};
MultiLangDaemon.MultiLangDaemonArguments arguments = new MultiLangDaemon.MultiLangDaemonArguments();
daemon.buildJCommanderAndParseArgs(arguments, args);
@ -124,7 +129,8 @@ public class MultiLangDaemonTest {
LoggerContext loggerContext = spy((LoggerContext) LoggerFactory.getILoggerFactory());
JoranConfigurator configurator = spy(new JoranConfigurator());
String logConfiguration = this.getClass().getClassLoader().getResource("logback.xml").getPath();
String logConfiguration =
this.getClass().getClassLoader().getResource("logback.xml").getPath();
daemon.configureLogging(logConfiguration, loggerContext, configurator);
verify(loggerContext).reset();

182
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MultiLangProtocolTest.java
View file

@ -14,6 +14,42 @@
*/
package software.amazon.kinesis.multilang;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import com.google.common.util.concurrent.SettableFuture;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.runners.MockitoJUnitRunner;
import org.mockito.stubbing.Answer;
import software.amazon.kinesis.exceptions.InvalidStateException;
import software.amazon.kinesis.exceptions.KinesisClientLibDependencyException;
import software.amazon.kinesis.exceptions.ShutdownException;
import software.amazon.kinesis.exceptions.ThrottlingException;
import software.amazon.kinesis.lifecycle.events.InitializationInput;
import software.amazon.kinesis.lifecycle.events.LeaseLostInput;
import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput;
import software.amazon.kinesis.lifecycle.events.ShardEndedInput;
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
import software.amazon.kinesis.multilang.messages.CheckpointMessage;
import software.amazon.kinesis.multilang.messages.LeaseLostMessage;
import software.amazon.kinesis.multilang.messages.Message;
import software.amazon.kinesis.multilang.messages.ProcessRecordsMessage;
import software.amazon.kinesis.multilang.messages.ShardEndedMessage;
import software.amazon.kinesis.multilang.messages.StatusMessage;
import software.amazon.kinesis.processor.RecordProcessorCheckpointer;
import software.amazon.kinesis.retrieval.KinesisClientRecord;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
@ -27,65 +63,35 @@ import static org.mockito.Mockito.timeout;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.runners.MockitoJUnitRunner;
import org.mockito.stubbing.Answer;
import software.amazon.kinesis.exceptions.InvalidStateException;
import software.amazon.kinesis.exceptions.KinesisClientLibDependencyException;
import software.amazon.kinesis.exceptions.ShutdownException;
import software.amazon.kinesis.exceptions.ThrottlingException;
import software.amazon.kinesis.lifecycle.events.LeaseLostInput;
import software.amazon.kinesis.lifecycle.events.ShardEndedInput;
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
import software.amazon.kinesis.multilang.messages.CheckpointMessage;
import software.amazon.kinesis.multilang.messages.LeaseLostMessage;
import software.amazon.kinesis.multilang.messages.Message;
import software.amazon.kinesis.multilang.messages.ProcessRecordsMessage;
import software.amazon.kinesis.multilang.messages.ShardEndedMessage;
import software.amazon.kinesis.multilang.messages.StatusMessage;
import com.google.common.util.concurrent.SettableFuture;
import software.amazon.kinesis.lifecycle.events.InitializationInput;
import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput;
import software.amazon.kinesis.processor.RecordProcessorCheckpointer;
import software.amazon.kinesis.retrieval.KinesisClientRecord;
@RunWith(MockitoJUnitRunner.class)
public class MultiLangProtocolTest {
private static final List<KinesisClientRecord> EMPTY_RECORD_LIST = Collections.emptyList();
@Mock
private MultiLangProtocol protocol;
@Mock
private MessageWriter messageWriter;
@Mock
private MessageReader messageReader;
private String shardId;
@Mock
private RecordProcessorCheckpointer checkpointer;
@Mock
private MultiLangDaemonConfiguration configuration;
@Before
public void setup() {
this.shardId = "shard-id-123";
protocol = new MultiLangProtocolForTesting(messageReader, messageWriter,
InitializationInput.builder().shardId(shardId).build(), configuration);
protocol = new MultiLangProtocolForTesting(
messageReader,
messageWriter,
InitializationInput.builder().shardId(shardId).build(),
configuration);
when(configuration.getTimeoutInSeconds()).thenReturn(null);
}
@ -104,28 +110,32 @@ public class MultiLangProtocolTest {
@Test
public void testInitialize() {
when(messageWriter
.writeInitializeMessage(argThat(Matchers.withInit(InitializationInput.builder()
.shardId(shardId).build())))).thenReturn(buildFuture(true));
when(messageReader.getNextMessageFromSTDOUT()).thenReturn(buildFuture(
new StatusMessage("initialize"), Message.class));
when(messageWriter.writeInitializeMessage(argThat(Matchers.withInit(
InitializationInput.builder().shardId(shardId).build()))))
.thenReturn(buildFuture(true));
when(messageReader.getNextMessageFromSTDOUT())
.thenReturn(buildFuture(new StatusMessage("initialize"), Message.class));
assertThat(protocol.initialize(), equalTo(true));
}
@Test
public void testProcessRecords() {
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class))).thenReturn(buildFuture(true));
when(messageReader.getNextMessageFromSTDOUT()).thenReturn(buildFuture(
new StatusMessage("processRecords"), Message.class));
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class)))
.thenReturn(buildFuture(true));
when(messageReader.getNextMessageFromSTDOUT())
.thenReturn(buildFuture(new StatusMessage("processRecords"), Message.class));
assertThat(protocol.processRecords(ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST).build()),
assertThat(
protocol.processRecords(
ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST).build()),
equalTo(true));
}
@Test
public void leaseLostTest() {
when(messageWriter.writeLeaseLossMessage(any(LeaseLostInput.class))).thenReturn(buildFuture(true));
when(messageReader.getNextMessageFromSTDOUT()).thenReturn(buildFuture(new StatusMessage(LeaseLostMessage.ACTION), Message.class));
when(messageReader.getNextMessageFromSTDOUT())
.thenReturn(buildFuture(new StatusMessage(LeaseLostMessage.ACTION), Message.class));
assertThat(protocol.leaseLost(LeaseLostInput.builder().build()), equalTo(true));
}
@ -133,7 +143,8 @@ public class MultiLangProtocolTest {
@Test
public void shardEndedTest() {
when(messageWriter.writeShardEndedMessage(any(ShardEndedInput.class))).thenReturn(buildFuture(true));
when(messageReader.getNextMessageFromSTDOUT()).thenReturn(buildFuture(new StatusMessage(ShardEndedMessage.ACTION)));
when(messageReader.getNextMessageFromSTDOUT())
.thenReturn(buildFuture(new StatusMessage(ShardEndedMessage.ACTION)));
assertThat(protocol.shardEnded(ShardEndedInput.builder().build()), equalTo(true));
}
@ -141,12 +152,12 @@ public class MultiLangProtocolTest {
@Test
public void shutdownRequestedTest() {
when(messageWriter.writeShutdownRequestedMessage()).thenReturn(buildFuture(true));
when(messageReader.getNextMessageFromSTDOUT()).thenReturn(buildFuture(
new StatusMessage("shutdownRequested"), Message.class));
Mockito.doReturn(buildFuture(true)).when(messageWriter)
.writeShutdownRequestedMessage();
when(messageReader.getNextMessageFromSTDOUT())
.thenReturn(buildFuture(new StatusMessage("shutdownRequested"), Message.class));
Mockito.doReturn(buildFuture(true)).when(messageWriter).writeShutdownRequestedMessage();
Mockito.doReturn(buildFuture(new StatusMessage("shutdownRequested")))
.when(messageReader).getNextMessageFromSTDOUT();
.when(messageReader)
.getNextMessageFromSTDOUT();
assertThat(protocol.shutdownRequested(null), equalTo(true));
}
@ -168,16 +179,17 @@ public class MultiLangProtocolTest {
}
return buildFuture(message);
}
}.init(messages);
}
@Test
public void testProcessRecordsWithCheckpoints() throws
KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
public void testProcessRecordsWithCheckpoints()
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class))).thenReturn(buildFuture(true));
when(messageWriter.writeCheckpointMessageWithError(anyString(), anyLong(), any(Throwable.class))).thenReturn(buildFuture(true));
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class)))
.thenReturn(buildFuture(true));
when(messageWriter.writeCheckpointMessageWithError(anyString(), anyLong(), any(Throwable.class)))
.thenReturn(buildFuture(true));
when(messageReader.getNextMessageFromSTDOUT()).thenAnswer(buildMessageAnswers(new ArrayList<Message>() {
{
this.add(new CheckpointMessage("123", 0L, null));
@ -192,8 +204,10 @@ public class MultiLangProtocolTest {
}
}));
boolean result = protocol.processRecords(ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST)
.checkpointer(checkpointer).build());
boolean result = protocol.processRecords(ProcessRecordsInput.builder()
.records(EMPTY_RECORD_LIST)
.checkpointer(checkpointer)
.build());
assertThat(result, equalTo(true));
@ -203,41 +217,52 @@ public class MultiLangProtocolTest {
@Test
public void testProcessRecordsWithABadCheckpoint() {
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class))).thenReturn(buildFuture(true));
when(messageWriter.writeCheckpointMessageWithError(anyString(), anyLong(), any(Throwable.class))).thenReturn(buildFuture(false));
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class)))
.thenReturn(buildFuture(true));
when(messageWriter.writeCheckpointMessageWithError(anyString(), anyLong(), any(Throwable.class)))
.thenReturn(buildFuture(false));
when(messageReader.getNextMessageFromSTDOUT()).thenAnswer(buildMessageAnswers(new ArrayList<Message>() {
{
this.add(new CheckpointMessage("456", 0L, null));
this.add(new StatusMessage("processRecords"));
}
}));
assertThat(protocol.processRecords(ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST)
.checkpointer(checkpointer).build()), equalTo(false));
assertThat(
protocol.processRecords(ProcessRecordsInput.builder()
.records(EMPTY_RECORD_LIST)
.checkpointer(checkpointer)
.build()),
equalTo(false));
}
@Test(expected = NullPointerException.class)
public void waitForStatusMessageTimeoutTest() throws InterruptedException, TimeoutException, ExecutionException {
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class))).thenReturn(buildFuture(true));
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class)))
.thenReturn(buildFuture(true));
Future<Message> future = Mockito.mock(Future.class);
when(messageReader.getNextMessageFromSTDOUT()).thenReturn(future);
when(configuration.getTimeoutInSeconds()).thenReturn(5);
when(future.get(anyInt(), eq(TimeUnit.SECONDS))).thenThrow(TimeoutException.class);
protocol = new MultiLangProtocolForTesting(messageReader,
protocol = new MultiLangProtocolForTesting(
messageReader,
messageWriter,
InitializationInput.builder().shardId(shardId).build(),
configuration);
protocol.processRecords(ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST).build());
protocol.processRecords(
ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST).build());
}
@Test
public void waitForStatusMessageSuccessTest() {
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class))).thenReturn(buildFuture(true));
when(messageReader.getNextMessageFromSTDOUT()).thenReturn(buildFuture(
new StatusMessage("processRecords"), Message.class));
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class)))
.thenReturn(buildFuture(true));
when(messageReader.getNextMessageFromSTDOUT())
.thenReturn(buildFuture(new StatusMessage("processRecords"), Message.class));
when(configuration.getTimeoutInSeconds()).thenReturn(5);
assertTrue(protocol.processRecords(ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST).build()));
assertTrue(protocol.processRecords(
ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST).build()));
}
private class MultiLangProtocolForTesting extends MultiLangProtocol {
@ -249,10 +274,11 @@ public class MultiLangProtocolTest {
* @param initializationInput
* @param configuration
*/
MultiLangProtocolForTesting(final MessageReader messageReader,
final MessageWriter messageWriter,
final InitializationInput initializationInput,
final MultiLangDaemonConfiguration configuration) {
MultiLangProtocolForTesting(
final MessageReader messageReader,
final MessageWriter messageWriter,
final InitializationInput initializationInput,
final MultiLangDaemonConfiguration configuration) {
super(messageReader, messageWriter, initializationInput, configuration);
}

20
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/NestedPropertyKeyTest.java
View file

@ -15,6 +15,10 @@
package software.amazon.kinesis.multilang;
import com.amazonaws.regions.Regions;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.verify;
@ -24,11 +28,6 @@ import static software.amazon.kinesis.multilang.NestedPropertyKey.ENDPOINT_REGIO
import static software.amazon.kinesis.multilang.NestedPropertyKey.EXTERNAL_ID;
import static software.amazon.kinesis.multilang.NestedPropertyKey.parse;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
@RunWith(MockitoJUnitRunner.class)
public class NestedPropertyKeyTest {
@ -95,11 +94,11 @@ public class NestedPropertyKeyTest {
@Test
public void testNonmatchingParameters() {
final String[] params = new String[] {
null,
"",
"hello world", // no nested key
"foo=bar", // nested key, but is not a recognized key
createKey(EXTERNAL_ID, "eid") + "=extra", // valid key made invalid by second '='
null,
"",
"hello world", // no nested key
"foo=bar", // nested key, but is not a recognized key
createKey(EXTERNAL_ID, "eid") + "=extra", // valid key made invalid by second '='
};
parse(mockProcessor, params);
verifyZeroInteractions(mockProcessor);
@ -108,5 +107,4 @@ public class NestedPropertyKeyTest {
private static String createKey(final NestedPropertyKey key, final String value) {
return key.getNestedKey() + "=" + value;
}
}

3
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/ReadSTDERRTaskTest.java
View file

@ -72,7 +72,8 @@ public class ReadSTDERRTaskTest {
try {
finishedCleanly = result.get();
} catch (InterruptedException | ExecutionException e) {
Assert.fail("Should have been able to get a result. The error should be handled during the call and result in false.");
Assert.fail(
"Should have been able to get a result. The error should be handled during the call and result in false.");
}
Assert.assertFalse("Reading a line should have thrown an exception", finishedCleanly);
}

12
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/StreamingShardRecordProcessorFactoryTest.java
View file

@ -16,12 +16,11 @@ package software.amazon.kinesis.multilang;
import org.junit.Assert;
import org.junit.Test;
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
import software.amazon.kinesis.processor.ShardRecordProcessor;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
import software.amazon.kinesis.processor.ShardRecordProcessor;
@RunWith(MockitoJUnitRunner.class)
public class StreamingShardRecordProcessorFactoryTest {
@ -31,10 +30,13 @@ public class StreamingShardRecordProcessorFactoryTest {
@Test
public void createProcessorTest() {
MultiLangRecordProcessorFactory factory = new MultiLangRecordProcessorFactory("somecommand", null, configuration);
MultiLangRecordProcessorFactory factory =
new MultiLangRecordProcessorFactory("somecommand", null, configuration);
ShardRecordProcessor processor = factory.shardRecordProcessor();
Assert.assertEquals("Should have constructed a StreamingRecordProcessor", MultiLangShardRecordProcessor.class,
Assert.assertEquals(
"Should have constructed a StreamingRecordProcessor",
MultiLangShardRecordProcessor.class,
processor.getClass());
}
}

90
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/StreamingShardRecordProcessorTest.java
View file

@ -14,15 +14,6 @@
*/
package software.amazon.kinesis.multilang;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Matchers.anyString;
import static org.mockito.Matchers.argThat;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Collections;
@ -32,6 +23,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@ -41,9 +33,6 @@ import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.runners.MockitoJUnitRunner;
import org.mockito.stubbing.Answer;
import com.fasterxml.jackson.databind.ObjectMapper;
import software.amazon.awssdk.services.kinesis.model.Record;
import software.amazon.kinesis.exceptions.KinesisClientLibDependencyException;
import software.amazon.kinesis.exceptions.ThrottlingException;
@ -61,6 +50,15 @@ import software.amazon.kinesis.processor.PreparedCheckpointer;
import software.amazon.kinesis.processor.RecordProcessorCheckpointer;
import software.amazon.kinesis.retrieval.KinesisClientRecord;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Matchers.anyString;
import static org.mockito.Matchers.argThat;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
public class StreamingShardRecordProcessorTest {
@ -70,6 +68,7 @@ public class StreamingShardRecordProcessorTest {
@Mock
private Future<Message> messageFuture;
@Mock
private Future<Boolean> trueFuture;
@ -81,14 +80,13 @@ public class StreamingShardRecordProcessorTest {
}
@Override
public void checkpoint(String sequenceNumber) throws KinesisClientLibDependencyException,
ThrottlingException, IllegalArgumentException {
public void checkpoint(String sequenceNumber)
throws KinesisClientLibDependencyException, ThrottlingException, IllegalArgumentException {
throw new UnsupportedOperationException();
}
@Override
public void checkpoint(Record record)
throws KinesisClientLibDependencyException, ThrottlingException {
public void checkpoint(Record record) throws KinesisClientLibDependencyException, ThrottlingException {
throw new UnsupportedOperationException();
}
@ -141,7 +139,8 @@ public class StreamingShardRecordProcessorTest {
}
@Override
public PreparedCheckpointer prepareCheckpoint(String sequenceNumber, long subSequenceNumber, byte[] applicationState)
public PreparedCheckpointer prepareCheckpoint(
String sequenceNumber, long subSequenceNumber, byte[] applicationState)
throws KinesisClientLibDependencyException, ThrottlingException, IllegalArgumentException {
throw new UnsupportedOperationException();
}
@ -178,8 +177,14 @@ public class StreamingShardRecordProcessorTest {
when(configuration.getTimeoutInSeconds()).thenReturn(null);
recordProcessor =
new MultiLangShardRecordProcessor(new ProcessBuilder(), executor, new ObjectMapper(), messageWriter,
messageReader, errorReader, configuration) {
new MultiLangShardRecordProcessor(
new ProcessBuilder(),
executor,
new ObjectMapper(),
messageWriter,
messageReader,
errorReader,
configuration) {
// Just don't do anything when we exit.
void exit() {
@ -203,9 +208,12 @@ public class StreamingShardRecordProcessorTest {
Mockito.doReturn(Mockito.mock(Future.class)).when(messageReader).drainSTDOUT();
Mockito.doReturn(true).when(trueFuture).get();
when(messageWriter.writeInitializeMessage(any(InitializationInput.class))).thenReturn(trueFuture);
when(messageWriter.writeCheckpointMessageWithError(anyString(), anyLong(), any(Throwable.class))).thenReturn(trueFuture);
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class))).thenReturn(trueFuture);
when(messageWriter.writeInitializeMessage(any(InitializationInput.class)))
.thenReturn(trueFuture);
when(messageWriter.writeCheckpointMessageWithError(anyString(), anyLong(), any(Throwable.class)))
.thenReturn(trueFuture);
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class)))
.thenReturn(trueFuture);
when(messageWriter.writeLeaseLossMessage(any(LeaseLostInput.class))).thenReturn(trueFuture);
}
@ -223,11 +231,16 @@ public class StreamingShardRecordProcessorTest {
List<KinesisClientRecord> testRecords = Collections.emptyList();
recordProcessor.initialize(InitializationInput.builder().shardId(SHARD_ID).build());
recordProcessor.processRecords(ProcessRecordsInput.builder().records(testRecords)
.checkpointer(unimplementedCheckpointer).build());
recordProcessor.processRecords(ProcessRecordsInput.builder().records(testRecords)
.checkpointer(unimplementedCheckpointer).build());
recordProcessor.initialize(
InitializationInput.builder().shardId(SHARD_ID).build());
recordProcessor.processRecords(ProcessRecordsInput.builder()
.records(testRecords)
.checkpointer(unimplementedCheckpointer)
.build());
recordProcessor.processRecords(ProcessRecordsInput.builder()
.records(testRecords)
.checkpointer(unimplementedCheckpointer)
.build());
recordProcessor.leaseLost(LeaseLostInput.builder().build());
}
@ -235,9 +248,12 @@ public class StreamingShardRecordProcessorTest {
public void processorPhasesTest() throws InterruptedException, ExecutionException {
Answer<StatusMessage> answer = new Answer<StatusMessage>() {
StatusMessage[] answers = new StatusMessage[] { new StatusMessage(InitializeMessage.ACTION),
new StatusMessage(ProcessRecordsMessage.ACTION), new StatusMessage(ProcessRecordsMessage.ACTION),
new StatusMessage(ShutdownMessage.ACTION) };
StatusMessage[] answers = new StatusMessage[] {
new StatusMessage(InitializeMessage.ACTION),
new StatusMessage(ProcessRecordsMessage.ACTION),
new StatusMessage(ProcessRecordsMessage.ACTION),
new StatusMessage(ShutdownMessage.ACTION)
};
int callCount = 0;
@ -268,9 +284,12 @@ public class StreamingShardRecordProcessorTest {
* This bad message will cause shutdown to not attempt to send a message. i.e. avoid encountering an
* exception.
*/
StatusMessage[] answers = new StatusMessage[] { new StatusMessage("Bad"),
new StatusMessage(ProcessRecordsMessage.ACTION), new StatusMessage(ProcessRecordsMessage.ACTION),
new StatusMessage(ShutdownMessage.ACTION) };
StatusMessage[] answers = new StatusMessage[] {
new StatusMessage("Bad"),
new StatusMessage(ProcessRecordsMessage.ACTION),
new StatusMessage(ProcessRecordsMessage.ACTION),
new StatusMessage(ShutdownMessage.ACTION)
};
int callCount = 0;
@ -286,8 +305,9 @@ public class StreamingShardRecordProcessorTest {
phases(answer);
verify(messageWriter).writeInitializeMessage(argThat(Matchers.withInit(InitializationInput.builder()
.shardId(SHARD_ID).build())));
verify(messageWriter)
.writeInitializeMessage(argThat(Matchers.withInit(
InitializationInput.builder().shardId(SHARD_ID).build())));
verify(messageWriter, times(2)).writeProcessRecordsMessage(any(ProcessRecordsInput.class));
verify(messageWriter, never()).writeLeaseLossMessage(any(LeaseLostInput.class));
Assert.assertEquals(1, systemExitCount);

11
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/auth/KclSTSAssumeRoleSessionCredentialsProviderTest.java
View file

@ -16,10 +16,10 @@ package software.amazon.kinesis.multilang.auth;
import java.util.Arrays;
import static org.junit.Assert.assertEquals;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
public class KclSTSAssumeRoleSessionCredentialsProviderTest {
private static final String ARN = "arn";
@ -31,7 +31,7 @@ public class KclSTSAssumeRoleSessionCredentialsProviderTest {
*/
@Test
public void testConstructorWithoutOptionalParams() {
new KclSTSAssumeRoleSessionCredentialsProvider(new String[] { ARN, SESSION_NAME });
new KclSTSAssumeRoleSessionCredentialsProvider(new String[] {ARN, SESSION_NAME});
}
@Test
@ -46,9 +46,8 @@ public class KclSTSAssumeRoleSessionCredentialsProviderTest {
@Test
public void testVarArgs() {
for (final String[] varargs : Arrays.asList(
new String[] { ARN, SESSION_NAME, "externalId=eid", "foo"},
new String[] { ARN, SESSION_NAME, "foo", "externalId=eid"}
)) {
new String[] {ARN, SESSION_NAME, "externalId=eid", "foo"},
new String[] {ARN, SESSION_NAME, "foo", "externalId=eid"})) {
final VarArgsSpy provider = new VarArgsSpy(varargs);
assertEquals("eid", provider.externalId);
}

37
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoderTest.java
View file

@ -14,14 +14,11 @@
*/
package software.amazon.kinesis.multilang.config;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
import java.util.Arrays;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSCredentialsProviderChain;
import com.amazonaws.auth.BasicAWSCredentials;
import lombok.ToString;
import org.hamcrest.Description;
@ -30,9 +27,11 @@ import org.hamcrest.TypeSafeDiagnosingMatcher;
import org.junit.Test;
import software.amazon.kinesis.multilang.auth.KclSTSAssumeRoleSessionCredentialsProvider;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSCredentialsProviderChain;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
public class AWSCredentialsProviderPropertyValueDecoderTest {
@ -79,10 +78,10 @@ public class AWSCredentialsProviderPropertyValueDecoderTest {
@Override
public void describeTo(Description description) {
description.appendText("An AWSCredentialsProvider that provides an AWSCredential matching: ")
description
.appendText("An AWSCredentialsProvider that provides an AWSCredential matching: ")
.appendList("(", ", ", ")", Arrays.asList(classMatcher, akidMatcher, secretMatcher));
}
}
private static AWSCredentialsMatcher hasCredentials(String akid, String secret) {
@ -121,7 +120,7 @@ public class AWSCredentialsProviderPropertyValueDecoderTest {
for (final String className : Arrays.asList(
KclSTSAssumeRoleSessionCredentialsProvider.class.getName(), // fully-qualified name
KclSTSAssumeRoleSessionCredentialsProvider.class.getSimpleName() // name-only; needs prefix
)) {
)) {
final AWSCredentialsProvider provider = decoder.decodeValue(className + "|arn|sessionName");
assertNotNull(className, provider);
}
@ -132,7 +131,7 @@ public class AWSCredentialsProviderPropertyValueDecoderTest {
*/
@Test
public void testVarArgAuthProvider() {
final String[] args = new String[] { "arg1", "arg2", "arg3" };
final String[] args = new String[] {"arg1", "arg2", "arg3"};
final String className = VarArgCredentialsProvider.class.getName();
final String encodedValue = className + "|" + String.join("|", args);
@ -151,9 +150,7 @@ public class AWSCredentialsProviderPropertyValueDecoderTest {
}
@Override
public void refresh() {
}
public void refresh() {}
}
/**
@ -180,9 +177,7 @@ public class AWSCredentialsProviderPropertyValueDecoderTest {
}
@Override
public void refresh() {
}
public void refresh() {}
}
private static class VarArgCredentialsProvider implements AWSCredentialsProvider {
@ -201,8 +196,6 @@ public class AWSCredentialsProviderPropertyValueDecoderTest {
}
@Override
public void refresh() {
}
public void refresh() {}
}
}

216
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/BuilderDynaBeanTest.java
View file

@ -15,16 +15,14 @@
package software.amazon.kinesis.multilang.config;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.CoreMatchers.nullValue;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.fail;
import java.util.function.Consumer;
import java.util.function.Supplier;
import lombok.Builder;
import lombok.EqualsAndHashCode;
import lombok.RequiredArgsConstructor;
import lombok.ToString;
import lombok.experimental.Accessors;
import org.apache.commons.beanutils.BeanUtilsBean;
import org.apache.commons.beanutils.ConvertUtilsBean;
import org.junit.Before;
@ -32,11 +30,12 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import lombok.Builder;
import lombok.EqualsAndHashCode;
import lombok.RequiredArgsConstructor;
import lombok.ToString;
import lombok.experimental.Accessors;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.CoreMatchers.nullValue;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.fail;
public class BuilderDynaBeanTest {
@ -109,8 +108,8 @@ public class BuilderDynaBeanTest {
@Test
public void testComplexCreateAllParameters() throws Exception {
TestComplexCreate expected = TestComplexCreate.create("real",
TestSimpleBuilder.builder().stringL1("l1").longVal(10L).build());
TestComplexCreate expected = TestComplexCreate.create(
"real", TestSimpleBuilder.builder().stringL1("l1").longVal(10L).build());
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreate.class, convertUtilsBean);
utilsBean.setProperty(builderDynaBean, "[0]", expected.realName);
@ -136,8 +135,8 @@ public class BuilderDynaBeanTest {
@Test
public void testComplexCreateComplexParameterOnly() throws Exception {
TestComplexCreate expected = TestComplexCreate.create(null,
TestSimpleBuilder.builder().stringL1("l1").longVal(10L).build());
TestComplexCreate expected = TestComplexCreate.create(
null, TestSimpleBuilder.builder().stringL1("l1").longVal(10L).build());
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreate.class, convertUtilsBean);
utilsBean.setProperty(builderDynaBean, "[1].stringL1", expected.test1.stringL1);
@ -161,7 +160,8 @@ public class BuilderDynaBeanTest {
@Test
public void testSimpleBuilderAllParameters() throws Exception {
TestSimpleBuilder expected = TestSimpleBuilder.builder().stringL1("l1").longVal(10L).build();
TestSimpleBuilder expected =
TestSimpleBuilder.builder().stringL1("l1").longVal(10L).build();
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleBuilder.class, convertUtilsBean);
utilsBean.setProperty(builderDynaBean, "stringL1", expected.stringL1);
@ -213,12 +213,14 @@ public class BuilderDynaBeanTest {
@Test
public void testComplexCreateSimpleBuilderVariantAllParameters() throws Exception {
TestSimpleBuilder variant = TestSimpleBuilder.builder().longVal(10L).stringL1("variant").build();
TestSimpleBuilder variant =
TestSimpleBuilder.builder().longVal(10L).stringL1("variant").build();
TestComplexCreateVariance expected = TestComplexCreateVariance.create("simple-builder", variant);
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean);
utilsBean.setProperty(builderDynaBean, "[0]", expected.varianceName);
utilsBean.setProperty(builderDynaBean, "[1].class", expected.variant.getClass().getName());
utilsBean.setProperty(
builderDynaBean, "[1].class", expected.variant.getClass().getName());
utilsBean.setProperty(builderDynaBean, "[1].longVal", variant.longVal);
utilsBean.setProperty(builderDynaBean, "[1].stringL1", variant.stringL1);
@ -229,8 +231,11 @@ public class BuilderDynaBeanTest {
@Test
public void testComplexCreateVariantBuilderAllParameters() throws Exception {
TestVariantBuilder variant = TestVariantBuilder.builder().variantBuilderName("variant-build").intClass(20)
.testEnum(TestEnum.Blue).build();
TestVariantBuilder variant = TestVariantBuilder.builder()
.variantBuilderName("variant-build")
.intClass(20)
.testEnum(TestEnum.Blue)
.build();
TestComplexCreateVariance expected = TestComplexCreateVariance.create("builder-variant", variant);
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean);
@ -264,13 +269,16 @@ public class BuilderDynaBeanTest {
@Test
public void testComplexCreateVariantBuilderAllParametersPrefixWithJoiner() throws Exception {
TestVariantBuilder variant = TestVariantBuilder.builder().variantBuilderName("variant-build").intClass(20)
.testEnum(TestEnum.Blue).build();
TestVariantBuilder variant = TestVariantBuilder.builder()
.variantBuilderName("variant-build")
.intClass(20)
.testEnum(TestEnum.Blue)
.build();
TestComplexCreateVariance expected = TestComplexCreateVariance.create("builder-variant-prefix", variant);
String prefix = variant.getClass().getEnclosingClass().getName() + "$";
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean,
prefix);
BuilderDynaBean builderDynaBean =
new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean, prefix);
utilsBean.setProperty(builderDynaBean, "[0]", expected.varianceName);
utilsBean.setProperty(builderDynaBean, "[1].class", variant.getClass().getSimpleName());
utilsBean.setProperty(builderDynaBean, "[1].variantBuilderName", variant.variantBuilderName);
@ -284,13 +292,16 @@ public class BuilderDynaBeanTest {
@Test
public void testComplexCreateVariantBuilderAllParametersPrefixWithOutJoiner() throws Exception {
TestVariantBuilder variant = TestVariantBuilder.builder().variantBuilderName("variant-build").intClass(20)
.testEnum(TestEnum.Blue).build();
TestVariantBuilder variant = TestVariantBuilder.builder()
.variantBuilderName("variant-build")
.intClass(20)
.testEnum(TestEnum.Blue)
.build();
TestComplexCreateVariance expected = TestComplexCreateVariance.create("builder-variant-prefix", variant);
String prefix = variant.getClass().getEnclosingClass().getName();
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean,
prefix);
BuilderDynaBean builderDynaBean =
new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean, prefix);
utilsBean.setProperty(builderDynaBean, "[0]", expected.varianceName);
utilsBean.setProperty(builderDynaBean, "[1].class", variant.getClass().getSimpleName());
utilsBean.setProperty(builderDynaBean, "[1].variantBuilderName", variant.variantBuilderName);
@ -330,11 +341,21 @@ public class BuilderDynaBeanTest {
@Test
public void testComplexRootAllParameters() throws Exception {
TestSimpleBuilder simpleBuilder = TestSimpleBuilder.builder().stringL1("simple-l1").longVal(20L).build();
TestRootClass expected = TestRootClass.builder().intVal(10).stringVal("root").testEnum(TestEnum.Red)
.testComplexCreate(TestComplexCreate.create("real",
TestSimpleBuilder.builder().stringL1("complex-l1").longVal(10L).build()))
.testSimpleBuilder(simpleBuilder).testSimpleCreate(TestSimpleCreate.create("first", "last")).build();
TestSimpleBuilder simpleBuilder =
TestSimpleBuilder.builder().stringL1("simple-l1").longVal(20L).build();
TestRootClass expected = TestRootClass.builder()
.intVal(10)
.stringVal("root")
.testEnum(TestEnum.Red)
.testComplexCreate(TestComplexCreate.create(
"real",
TestSimpleBuilder.builder()
.stringL1("complex-l1")
.longVal(10L)
.build()))
.testSimpleBuilder(simpleBuilder)
.testSimpleCreate(TestSimpleCreate.create("first", "last"))
.build();
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestRootClass.class, convertUtilsBean);
@ -342,10 +363,10 @@ public class BuilderDynaBeanTest {
utilsBean.setProperty(builderDynaBean, "stringVal", expected.stringVal);
utilsBean.setProperty(builderDynaBean, "testEnum", expected.testEnum);
utilsBean.setProperty(builderDynaBean, "testComplexCreate.[0]", expected.testComplexCreate.realName);
utilsBean.setProperty(builderDynaBean, "testComplexCreate.[1].stringL1",
expected.testComplexCreate.test1.stringL1);
utilsBean.setProperty(builderDynaBean, "testComplexCreate.[1].longVal",
expected.testComplexCreate.test1.longVal);
utilsBean.setProperty(
builderDynaBean, "testComplexCreate.[1].stringL1", expected.testComplexCreate.test1.stringL1);
utilsBean.setProperty(
builderDynaBean, "testComplexCreate.[1].longVal", expected.testComplexCreate.test1.longVal);
utilsBean.setProperty(builderDynaBean, "testSimpleBuilder.class", TestSimpleBuilder.class.getName());
utilsBean.setProperty(builderDynaBean, "testSimpleBuilder.stringL1", simpleBuilder.stringL1);
utilsBean.setProperty(builderDynaBean, "testSimpleBuilder.longVal", simpleBuilder.longVal);
@ -370,7 +391,11 @@ public class BuilderDynaBeanTest {
@Test
public void testComplexRootTopLevelOnly() throws Exception {
TestRootClass expected = TestRootClass.builder().intVal(10).stringVal("root").testEnum(TestEnum.Red).build();
TestRootClass expected = TestRootClass.builder()
.intVal(10)
.stringVal("root")
.testEnum(TestEnum.Red)
.build();
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestRootClass.class, convertUtilsBean);
@ -385,12 +410,17 @@ public class BuilderDynaBeanTest {
@Test
public void testSupplierNotUsed() throws Exception {
TestVariantBuilder variant = TestVariantBuilder.builder().testEnum(TestEnum.Green).intClass(10)
.variantBuilderName("variant-supplier").build();
TestSupplierClass expected = TestSupplierClass.builder().variantClass(variant).build();
TestVariantBuilder variant = TestVariantBuilder.builder()
.testEnum(TestEnum.Green)
.intClass(10)
.variantBuilderName("variant-supplier")
.build();
TestSupplierClass expected =
TestSupplierClass.builder().variantClass(variant).build();
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSupplierClass.class, convertUtilsBean);
utilsBean.setProperty(builderDynaBean, "variantClass.class", variant.getClass().getName());
utilsBean.setProperty(
builderDynaBean, "variantClass.class", variant.getClass().getName());
utilsBean.setProperty(builderDynaBean, "variantClass.testEnum", variant.testEnum);
utilsBean.setProperty(builderDynaBean, "variantClass.intClass", variant.intClass);
utilsBean.setProperty(builderDynaBean, "variantClass.variantBuilderName", variant.variantBuilderName);
@ -422,8 +452,11 @@ public class BuilderDynaBeanTest {
@Test
public void testVariantBuildsToSuperType() throws Exception {
TestVariantBuilder expected = TestVariantBuilder.builder().intClass(10).testEnum(TestEnum.Green)
.variantBuilderName("variant-super").build();
TestVariantBuilder expected = TestVariantBuilder.builder()
.intClass(10)
.testEnum(TestEnum.Green)
.variantBuilderName("variant-super")
.build();
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestInterface.class, convertUtilsBean);
utilsBean.setProperty(builderDynaBean, "class", expected.getClass().getName());
@ -439,9 +472,11 @@ public class BuilderDynaBeanTest {
@Test
public void testEmptyPropertyHandler() throws Exception {
String emptyPropertyValue = "test-property";
TestVariantCreate expected = TestVariantCreate.create(emptyPropertyValue, (long) emptyPropertyValue.length(),
emptyPropertyValue + "-vary");
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestInterface.class, convertUtilsBean,
TestVariantCreate expected = TestVariantCreate.create(
emptyPropertyValue, (long) emptyPropertyValue.length(), emptyPropertyValue + "-vary");
BuilderDynaBean builderDynaBean = new BuilderDynaBean(
TestInterface.class,
convertUtilsBean,
s -> TestVariantCreate.create(s, (long) s.length(), s + "-vary"));
utilsBean.setProperty(builderDynaBean, "", emptyPropertyValue);
@ -455,8 +490,8 @@ public class BuilderDynaBeanTest {
thrown.expect(IllegalStateException.class);
thrown.expectMessage(containsString("When a property handler is resolved further properties may not be set."));
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestInterface.class, convertUtilsBean,
s -> TestVariantCreate.create("test", 10, "test"));
BuilderDynaBean builderDynaBean = new BuilderDynaBean(
TestInterface.class, convertUtilsBean, s -> TestVariantCreate.create("test", 10, "test"));
utilsBean.setProperty(builderDynaBean, "", "test");
utilsBean.setProperty(builderDynaBean, "[0]", "test");
}
@ -468,8 +503,8 @@ public class BuilderDynaBeanTest {
thrown.expectMessage(containsString(TestInterface.class.getName()));
thrown.expectMessage(containsString("cannot be assigned to"));
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestInterface.class, convertUtilsBean,
s -> TestEnum.Green);
BuilderDynaBean builderDynaBean =
new BuilderDynaBean(TestInterface.class, convertUtilsBean, s -> TestEnum.Green);
utilsBean.setProperty(builderDynaBean, "", "test");
@ -478,8 +513,11 @@ public class BuilderDynaBeanTest {
@Test
public void testSimpleArrayValues() throws Exception {
SimpleArrayClassVariant expected = SimpleArrayClassVariant.builder().ints(new Integer[] { 1, 2, 3 })
.variantName("simple-array").longs(new Long[] { 1L, 2L, 3L }).strings(new String[] { "a", "b", "c" })
SimpleArrayClassVariant expected = SimpleArrayClassVariant.builder()
.ints(new Integer[] {1, 2, 3})
.variantName("simple-array")
.longs(new Long[] {1L, 2L, 3L})
.strings(new String[] {"a", "b", "c"})
.build();
BuilderDynaBean builderDynaBean = new BuilderDynaBean(SimpleArrayClassVariant.class, convertUtilsBean);
@ -503,12 +541,20 @@ public class BuilderDynaBeanTest {
@Test
public void testComplexArrayValuesBuilder() throws Exception {
TestVariantBuilder variant1 = TestVariantBuilder.builder().variantBuilderName("variant-1")
.testEnum(TestEnum.Green).intClass(10).build();
TestVariantBuilder variant2 = TestVariantBuilder.builder().variantBuilderName("variant-2")
.testEnum(TestEnum.Blue).intClass(20).build();
ComplexArrayClassVariant expected = ComplexArrayClassVariant.builder().variantName("complex-test")
.tests(new TestInterface[] { variant1, variant2 }).build();
TestVariantBuilder variant1 = TestVariantBuilder.builder()
.variantBuilderName("variant-1")
.testEnum(TestEnum.Green)
.intClass(10)
.build();
TestVariantBuilder variant2 = TestVariantBuilder.builder()
.variantBuilderName("variant-2")
.testEnum(TestEnum.Blue)
.intClass(20)
.build();
ComplexArrayClassVariant expected = ComplexArrayClassVariant.builder()
.variantName("complex-test")
.tests(new TestInterface[] {variant1, variant2})
.build();
BuilderDynaBean builderDynaBean = new BuilderDynaBean(ComplexArrayClassVariant.class, convertUtilsBean);
@ -533,18 +579,22 @@ public class BuilderDynaBeanTest {
TestVariantCreate variant1 = TestVariantCreate.create("variant-1", 10L, "vary-1");
TestVariantCreate variant2 = TestVariantCreate.create("variant-2", 20L, "vary-2");
ComplexArrayClassVariant expected = ComplexArrayClassVariant.builder().variantName("create-test")
.tests(new TestInterface[] { variant1, variant2 }).build();
ComplexArrayClassVariant expected = ComplexArrayClassVariant.builder()
.variantName("create-test")
.tests(new TestInterface[] {variant1, variant2})
.build();
BuilderDynaBean builderDynaBean = new BuilderDynaBean(ComplexArrayClassVariant.class, convertUtilsBean);
utilsBean.setProperty(builderDynaBean, "variantName", expected.variantName);
utilsBean.setProperty(builderDynaBean, "tests[0].class", variant1.getClass().getName());
utilsBean.setProperty(
builderDynaBean, "tests[0].class", variant1.getClass().getName());
utilsBean.setProperty(builderDynaBean, "tests[0].[0]", variant1.variantCreateName);
utilsBean.setProperty(builderDynaBean, "tests[0].[1]", variant1.longClass);
utilsBean.setProperty(builderDynaBean, "tests[0].[2]", variant1.varyString);
utilsBean.setProperty(builderDynaBean, "tests[1].class", variant2.getClass().getName());
utilsBean.setProperty(
builderDynaBean, "tests[1].class", variant2.getClass().getName());
utilsBean.setProperty(builderDynaBean, "tests[1].[0]", variant2.variantCreateName);
utilsBean.setProperty(builderDynaBean, "tests[1].[1]", variant2.longClass);
utilsBean.setProperty(builderDynaBean, "tests[1].[2]", variant2.varyString);
@ -552,7 +602,6 @@ public class BuilderDynaBeanTest {
ComplexArrayClassVariant actual = builderDynaBean.build(ComplexArrayClassVariant.class);
assertThat(actual, equalTo(expected));
}
@Test
@ -562,13 +611,18 @@ public class BuilderDynaBeanTest {
if (i % 2 == 0) {
variants[i] = TestVariantCreate.create("create-variant-" + i, i + 5, "vary-" + i);
} else {
variants[i] = TestVariantBuilder.builder().testEnum(TestEnum.values()[i % TestEnum.values().length])
.intClass(i).variantBuilderName("builder-variant-" + i).build();
variants[i] = TestVariantBuilder.builder()
.testEnum(TestEnum.values()[i % TestEnum.values().length])
.intClass(i)
.variantBuilderName("builder-variant-" + i)
.build();
}
}
ComplexArrayClassVariant expected = ComplexArrayClassVariant.builder().variantName("large-complex")
.tests(variants).build();
ComplexArrayClassVariant expected = ComplexArrayClassVariant.builder()
.variantName("large-complex")
.tests(variants)
.build();
BuilderDynaBean builderDynaBean = new BuilderDynaBean(ComplexArrayClassVariant.class, convertUtilsBean);
@ -578,13 +632,15 @@ public class BuilderDynaBeanTest {
TestInterface variant = variants[i];
if (variant instanceof TestVariantCreate) {
TestVariantCreate create = (TestVariantCreate) variant;
utilsBean.setProperty(builderDynaBean, prefix + "class", create.getClass().getName());
utilsBean.setProperty(
builderDynaBean, prefix + "class", create.getClass().getName());
utilsBean.setProperty(builderDynaBean, prefix + "[0]", create.variantCreateName);
utilsBean.setProperty(builderDynaBean, prefix + "[1]", create.longClass);
utilsBean.setProperty(builderDynaBean, prefix + "[2]", create.varyString);
} else if (variant instanceof TestVariantBuilder) {
TestVariantBuilder builder = (TestVariantBuilder) variant;
utilsBean.setProperty(builderDynaBean, prefix + "class", builder.getClass().getName());
utilsBean.setProperty(
builderDynaBean, prefix + "class", builder.getClass().getName());
utilsBean.setProperty(builderDynaBean, prefix + "variantBuilderName", builder.variantBuilderName);
utilsBean.setProperty(builderDynaBean, prefix + "intClass", builder.intClass);
utilsBean.setProperty(builderDynaBean, prefix + "testEnum", builder.testEnum);
@ -667,25 +723,27 @@ public class BuilderDynaBeanTest {
@Test
public void testAdditionalMutators() throws Exception {
TestSimpleBuilder expected = TestSimpleBuilder.builder().stringL1("test").longVal(10L).build();
TestSimpleBuilder expected =
TestSimpleBuilder.builder().stringL1("test").longVal(10L).build();
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleBuilder.class, convertUtilsBean);
utilsBean.setProperty(builderDynaBean, "stringL1", expected.stringL1);
TestSimpleBuilder actual = builderDynaBean.build(TestSimpleBuilder.class,
b -> ((TestSimpleBuilder.TestSimpleBuilderBuilder) b).longVal(expected.longVal));
TestSimpleBuilder actual =
builderDynaBean.build(TestSimpleBuilder.class, b -> ((TestSimpleBuilder.TestSimpleBuilderBuilder) b)
.longVal(expected.longVal));
assertThat(actual, equalTo(expected));
}
public enum TestEnum {
Red, Green, Blue
Red,
Green,
Blue
}
public interface TestInterface {
}
public interface TestInterface {}
@Accessors(fluent = true)
@ToString
@ -838,7 +896,5 @@ public class BuilderDynaBeanTest {
}
public String name = "default";
}
}

100
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/ConfigurationSettableUtilsTest.java
View file

@ -15,18 +15,17 @@
package software.amazon.kinesis.multilang.config;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.junit.Assert.assertThat;
import java.util.Optional;
import org.junit.Test;
import lombok.Builder;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.Setter;
import lombok.experimental.Accessors;
import org.junit.Test;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.junit.Assert.assertThat;
public class ConfigurationSettableUtilsTest {
@ -44,7 +43,10 @@ public class ConfigurationSettableUtilsTest {
public void testPrimitivesSet() {
ConfigResult expected = ConfigResult.builder().rawInt(10).rawLong(15L).build();
ConfigObject configObject = ConfigObject.builder().rawInt(expected.rawInt).rawLong(expected.rawLong).build();
ConfigObject configObject = ConfigObject.builder()
.rawInt(expected.rawInt)
.rawLong(expected.rawLong)
.build();
ConfigResult actual = resolve(configObject);
assertThat(actual, equalTo(expected));
@ -52,10 +54,14 @@ public class ConfigurationSettableUtilsTest {
@Test
public void testHeapValuesSet() {
ConfigResult expected = ConfigResult.builder().name("test").boxedInt(10).boxedLong(15L).build();
ConfigResult expected =
ConfigResult.builder().name("test").boxedInt(10).boxedLong(15L).build();
ConfigObject configObject = ConfigObject.builder().name(expected.name).boxedInt(expected.boxedInt.intValue())
.boxedLong(expected.boxedLong.longValue()).build();
ConfigObject configObject = ConfigObject.builder()
.name(expected.name)
.boxedInt(expected.boxedInt.intValue())
.boxedLong(expected.boxedLong.longValue())
.build();
ConfigResult actual = resolve(configObject);
assertThat(actual, equalTo(expected));
@ -63,27 +69,39 @@ public class ConfigurationSettableUtilsTest {
@Test
public void testComplexValuesSet() {
ComplexValue complexValue = ComplexValue.builder().name("complex").value(10).build();
ConfigResult expected = ConfigResult.builder().complexValue(complexValue).build();
ComplexValue complexValue =
ComplexValue.builder().name("complex").value(10).build();
ConfigResult expected =
ConfigResult.builder().complexValue(complexValue).build();
ConfigObject configObject = ConfigObject.builder()
.complexValue(ComplexValue.builder().name(complexValue.name).value(complexValue.value).build()).build();
.complexValue(ComplexValue.builder()
.name(complexValue.name)
.value(complexValue.value)
.build())
.build();
ConfigResult actual = resolve(configObject);
assertThat(actual, equalTo(expected));
}
@Test
public void testOptionalValuesSet() {
ComplexValue complexValue = ComplexValue.builder().name("optional-complex").value(20).build();
ConfigResult expected = ConfigResult.builder().optionalString(Optional.of("test"))
.optionalInteger(Optional.of(10)).optionalLong(Optional.of(15L))
.optionalComplexValue(Optional.of(complexValue)).build();
ComplexValue complexValue =
ComplexValue.builder().name("optional-complex").value(20).build();
ConfigResult expected = ConfigResult.builder()
.optionalString(Optional.of("test"))
.optionalInteger(Optional.of(10))
.optionalLong(Optional.of(15L))
.optionalComplexValue(Optional.of(complexValue))
.build();
ConfigObject configObject = ConfigObject.builder().optionalString(expected.optionalString.get())
.optionalInteger(expected.optionalInteger.get()).optionalLong(expected.optionalLong.get())
.optionalComplexValue(expected.optionalComplexValue.get()).build();
ConfigObject configObject = ConfigObject.builder()
.optionalString(expected.optionalString.get())
.optionalInteger(expected.optionalInteger.get())
.optionalLong(expected.optionalLong.get())
.optionalComplexValue(expected.optionalComplexValue.get())
.build();
ConfigResult actual = resolve(configObject);
assertThat(actual, equalTo(expected));
@ -91,20 +109,29 @@ public class ConfigurationSettableUtilsTest {
@Test
public void testRenamedRawValues() {
ComplexValue complexValue = ComplexValue.builder().name("renamed-complex").value(20).build();
ConfigResult expected = ConfigResult.builder().renamedString("renamed").renamedInt(10)
.renamedOptionalString(Optional.of("renamed-optional")).renamedComplexValue(complexValue).build();
ComplexValue complexValue =
ComplexValue.builder().name("renamed-complex").value(20).build();
ConfigResult expected = ConfigResult.builder()
.renamedString("renamed")
.renamedInt(10)
.renamedOptionalString(Optional.of("renamed-optional"))
.renamedComplexValue(complexValue)
.build();
ConfigObject configObject = ConfigObject.builder().toRenameString(expected.renamedString)
.toRenameInt(expected.renamedInt).toRenameComplexValue(complexValue)
.optionalToRename(expected.renamedOptionalString.get()).build();
ConfigObject configObject = ConfigObject.builder()
.toRenameString(expected.renamedString)
.toRenameInt(expected.renamedInt)
.toRenameComplexValue(complexValue)
.optionalToRename(expected.renamedOptionalString.get())
.build();
ConfigResult actual = resolve(configObject);
assertThat(actual, equalTo(expected));
}
private ConfigResult resolve(ConfigObject configObject) {
return ConfigurationSettableUtils.resolveFields(configObject, ConfigResult.builder().build());
return ConfigurationSettableUtils.resolveFields(
configObject, ConfigResult.builder().build());
}
@Accessors(fluent = true)
@ -129,7 +156,6 @@ public class ConfigurationSettableUtilsTest {
private int renamedInt;
private Optional<String> renamedOptionalString;
private ComplexValue renamedComplexValue;
}
@Accessors(fluent = true)
@ -145,35 +171,47 @@ public class ConfigurationSettableUtilsTest {
@ConfigurationSettable(configurationClass = ConfigResult.class)
private String name;
@ConfigurationSettable(configurationClass = ConfigResult.class)
private int rawInt;
@ConfigurationSettable(configurationClass = ConfigResult.class)
private Integer boxedInt;
@ConfigurationSettable(configurationClass = ConfigResult.class)
private long rawLong;
@ConfigurationSettable(configurationClass = ConfigResult.class)
private Long boxedLong;
@ConfigurationSettable(configurationClass = ConfigResult.class)
private ComplexValue complexValue;
@ConfigurationSettable(configurationClass = ConfigResult.class, convertToOptional = true)
private String optionalString;
@ConfigurationSettable(configurationClass = ConfigResult.class, convertToOptional = true)
private Integer optionalInteger;
@ConfigurationSettable(configurationClass = ConfigResult.class, convertToOptional = true)
private Long optionalLong;
@ConfigurationSettable(configurationClass = ConfigResult.class, convertToOptional = true)
private ComplexValue optionalComplexValue;
@ConfigurationSettable(configurationClass = ConfigResult.class, methodName = "renamedString")
private String toRenameString;
@ConfigurationSettable(configurationClass = ConfigResult.class, methodName = "renamedInt")
private int toRenameInt;
@ConfigurationSettable(configurationClass = ConfigResult.class, methodName = "renamedOptionalString", convertToOptional = true)
@ConfigurationSettable(
configurationClass = ConfigResult.class,
methodName = "renamedOptionalString",
convertToOptional = true)
private String optionalToRename;
@ConfigurationSettable(configurationClass = ConfigResult.class, methodName = "renamedComplexValue")
private ComplexValue toRenameComplexValue;
}
}

4
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/DatePropertyValueDecoderTest.java
View file

@ -14,12 +14,12 @@
*/
package software.amazon.kinesis.multilang.config;
import static org.junit.Assert.assertEquals;
import java.util.Date;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
public class DatePropertyValueDecoderTest {
private DatePropertyValueDecoder decoder = new DatePropertyValueDecoder();

17
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/FanoutConfigBeanTest.java
View file

@ -21,7 +21,6 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
import software.amazon.awssdk.services.kinesis.KinesisAsyncClient;
import software.amazon.kinesis.retrieval.fanout.FanOutConfig;
@ -50,18 +49,22 @@ public class FanoutConfigBeanTest {
MultiLangDaemonConfiguration configuration = new MultiLangDaemonConfiguration(utilsBean, convertUtilsBean);
configuration.setStreamName("test-stream");
configuration.setApplicationName("test-application");
FanOutConfig fanOutConfig =fanoutConfigBean.build(kinesisAsyncClient, configuration);
FanOutConfig fanOutConfig = fanoutConfigBean.build(kinesisAsyncClient, configuration);
assertThat(fanOutConfig.kinesisClient(), equalTo(kinesisAsyncClient));
assertThat(fanOutConfig.streamName(), equalTo(configuration.getStreamName()));
assertThat(fanOutConfig.applicationName(), equalTo(configuration.getApplicationName()));
assertThat(fanOutConfig.consumerArn(), equalTo(fanoutConfigBean.getConsumerArn()));
assertThat(fanOutConfig.consumerName(), equalTo(fanoutConfigBean.getConsumerName()));
assertThat(fanOutConfig.maxDescribeStreamConsumerRetries(), equalTo(fanoutConfigBean.getMaxDescribeStreamConsumerRetries()));
assertThat(fanOutConfig.maxDescribeStreamSummaryRetries(), equalTo(fanoutConfigBean.getMaxDescribeStreamSummaryRetries()));
assertThat(fanOutConfig.registerStreamConsumerRetries(), equalTo(fanoutConfigBean.getRegisterStreamConsumerRetries()));
assertThat(
fanOutConfig.maxDescribeStreamConsumerRetries(),
equalTo(fanoutConfigBean.getMaxDescribeStreamConsumerRetries()));
assertThat(
fanOutConfig.maxDescribeStreamSummaryRetries(),
equalTo(fanoutConfigBean.getMaxDescribeStreamSummaryRetries()));
assertThat(
fanOutConfig.registerStreamConsumerRetries(),
equalTo(fanoutConfigBean.getRegisterStreamConsumerRetries()));
assertThat(fanOutConfig.retryBackoffMillis(), equalTo(fanoutConfigBean.getRetryBackoffMillis()));
}
}

383
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/KinesisClientLibConfiguratorTest.java
View file

@ -14,15 +14,6 @@
*/
package software.amazon.kinesis.multilang.config;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.nullValue;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.net.URI;
@ -34,18 +25,25 @@ import java.util.Set;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.google.common.collect.ImmutableSet;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.junit.Test;
import com.google.common.collect.ImmutableSet;
import org.junit.runner.RunWith;
import org.mockito.runners.MockitoJUnitRunner;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.kinesis.common.InitialPositionInStream;
import software.amazon.kinesis.metrics.MetricsLevel;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.nullValue;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@RunWith(MockitoJUnitRunner.class)
public class KinesisClientLibConfiguratorTest {
@ -58,8 +56,14 @@ public class KinesisClientLibConfiguratorTest {
@Test
public void testWithBasicSetup() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a",
"applicationName = b", "AWSCredentialsProvider = " + credentialName1, "workerId = 123" }, '\n'));
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = " + credentialName1,
"workerId = 123"
},
'\n'));
assertEquals(config.getApplicationName(), "b");
assertEquals(config.getStreamName(), "a");
assertEquals(config.getWorkerIdentifier(), "123");
@ -69,9 +73,16 @@ public class KinesisClientLibConfiguratorTest {
@Test
public void testWithLongVariables() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "applicationName = app",
"streamName = 123", "AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"workerId = 123", "failoverTimeMillis = 100", "shardSyncIntervalMillis = 500" }, '\n'));
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
new String[] {
"applicationName = app",
"streamName = 123",
"AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"workerId = 123",
"failoverTimeMillis = 100",
"shardSyncIntervalMillis = 500"
},
'\n'));
assertEquals(config.getApplicationName(), "app");
assertEquals(config.getStreamName(), "123");
@ -83,9 +94,14 @@ public class KinesisClientLibConfiguratorTest {
@Test
public void testWithInitialPositionInStreamExtended() {
long epochTimeInSeconds = 1617406032;
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "applicationName = app",
"streamName = 123", "AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"initialPositionInStreamExtended = " + epochTimeInSeconds}, '\n'));
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
new String[] {
"applicationName = app",
"streamName = 123",
"AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"initialPositionInStreamExtended = " + epochTimeInSeconds
},
'\n'));
assertEquals(config.getInitialPositionInStreamExtended().getTimestamp(), new Date(epochTimeInSeconds * 1000L));
assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.AT_TIMESTAMP);
@ -96,9 +112,14 @@ public class KinesisClientLibConfiguratorTest {
// AT_TIMESTAMP cannot be used as initialPositionInStream. If a user wants to specify AT_TIMESTAMP,
// they must specify the time with initialPositionInStreamExtended.
try {
getConfiguration(StringUtils.join(new String[] { "applicationName = app",
"streamName = 123", "AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"initialPositionInStream = AT_TIMESTAMP"}, '\n'));
getConfiguration(StringUtils.join(
new String[] {
"applicationName = app",
"streamName = 123",
"AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"initialPositionInStream = AT_TIMESTAMP"
},
'\n'));
fail("Should have thrown when initialPositionInStream is set to AT_TIMESTAMP");
} catch (Exception e) {
Throwable rootCause = ExceptionUtils.getRootCause(e);
@ -111,9 +132,14 @@ public class KinesisClientLibConfiguratorTest {
// initialPositionInStreamExtended takes a long value indicating seconds since epoch. If a non-long
// value is provided, the constructor should throw an IllegalArgumentException exception.
try {
getConfiguration(StringUtils.join(new String[] { "applicationName = app",
"streamName = 123", "AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"initialPositionInStreamExtended = null"}, '\n'));
getConfiguration(StringUtils.join(
new String[] {
"applicationName = app",
"streamName = 123",
"AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"initialPositionInStreamExtended = null"
},
'\n'));
fail("Should have thrown when initialPositionInStreamExtended is set to null");
} catch (Exception e) {
Throwable rootCause = ExceptionUtils.getRootCause(e);
@ -124,8 +150,13 @@ public class KinesisClientLibConfiguratorTest {
@Test
public void testWithUnsupportedClientConfigurationVariables() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
new String[] { "AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2, "workerId = id",
"kinesisClientConfig = {}", "streamName = stream", "applicationName = b" },
new String[] {
"AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"workerId = id",
"kinesisClientConfig = {}",
"streamName = stream",
"applicationName = b"
},
'\n'));
assertEquals(config.getApplicationName(), "b");
@ -136,10 +167,18 @@ public class KinesisClientLibConfiguratorTest {
@Test
public void testWithIntVariables() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = kinesis",
"AWSCredentialsProvider = " + credentialName2 + ", " + credentialName1, "workerId = w123",
"maxRecords = 10", "metricsMaxQueueSize = 20", "applicationName = kinesis",
"retryGetRecordsInSeconds = 2", "maxGetRecordsThreadPool = 1" }, '\n'));
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
new String[] {
"streamName = kinesis",
"AWSCredentialsProvider = " + credentialName2 + ", " + credentialName1,
"workerId = w123",
"maxRecords = 10",
"metricsMaxQueueSize = 20",
"applicationName = kinesis",
"retryGetRecordsInSeconds = 2",
"maxGetRecordsThreadPool = 1"
},
'\n'));
assertEquals(config.getApplicationName(), "kinesis");
assertEquals(config.getStreamName(), "kinesis");
@ -152,9 +191,15 @@ public class KinesisClientLibConfiguratorTest {
@Test
public void testWithBooleanVariables() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a",
"applicationName = b", "AWSCredentialsProvider = ABCD, " + credentialName1, "workerId = 0",
"cleanupLeasesUponShardCompletion = false", "validateSequenceNumberBeforeCheckpointing = true" },
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = ABCD, " + credentialName1,
"workerId = 0",
"cleanupLeasesUponShardCompletion = false",
"validateSequenceNumberBeforeCheckpointing = true"
},
'\n'));
assertEquals(config.getApplicationName(), "b");
@ -166,9 +211,16 @@ public class KinesisClientLibConfiguratorTest {
@Test
public void testWithStringVariables() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a",
"applicationName = b", "AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 1",
"kinesisEndpoint = https://kinesis", "metricsLevel = SUMMARY" }, '\n'));
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = ABCD," + credentialName1,
"workerId = 1",
"kinesisEndpoint = https://kinesis",
"metricsLevel = SUMMARY"
},
'\n'));
assertEquals(config.getWorkerIdentifier(), "1");
assertEquals(config.getKinesisClient().get("endpointOverride"), URI.create("https://kinesis"));
@ -177,38 +229,66 @@ public class KinesisClientLibConfiguratorTest {
@Test
public void testWithSetVariables() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a",
"applicationName = b", "AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 1",
"metricsEnabledDimensions = ShardId, WorkerIdentifier" }, '\n'));
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = ABCD," + credentialName1,
"workerId = 1",
"metricsEnabledDimensions = ShardId, WorkerIdentifier"
},
'\n'));
Set<String> expectedMetricsEnabledDimensions = ImmutableSet.<String> builder()
.add("ShardId", "WorkerIdentifier").build();
assertThat(new HashSet<>(Arrays.asList(config.getMetricsEnabledDimensions())), equalTo(expectedMetricsEnabledDimensions));
Set<String> expectedMetricsEnabledDimensions = ImmutableSet.<String>builder()
.add("ShardId", "WorkerIdentifier")
.build();
assertThat(
new HashSet<>(Arrays.asList(config.getMetricsEnabledDimensions())),
equalTo(expectedMetricsEnabledDimensions));
}
@Test
public void testWithInitialPositionInStreamTrimHorizon() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a",
"applicationName = b", "AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 123",
"initialPositionInStream = TriM_Horizon" }, '\n'));
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = ABCD," + credentialName1,
"workerId = 123",
"initialPositionInStream = TriM_Horizon"
},
'\n'));
assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.TRIM_HORIZON);
}
@Test
public void testWithInitialPositionInStreamLatest() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a",
"applicationName = b", "AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 123",
"initialPositionInStream = LateSt" }, '\n'));
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = ABCD," + credentialName1,
"workerId = 123",
"initialPositionInStream = LateSt"
},
'\n'));
assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.LATEST);
}
@Test
public void testSkippingNonKCLVariables() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a",
"applicationName = b", "AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 123",
"initialPositionInStream = TriM_Horizon", "abc = 1" }, '\n'));
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = ABCD," + credentialName1,
"workerId = 123",
"initialPositionInStream = TriM_Horizon",
"abc = 1"
},
'\n'));
assertEquals(config.getApplicationName(), "b");
assertEquals(config.getStreamName(), "a");
@ -218,33 +298,61 @@ public class KinesisClientLibConfiguratorTest {
@Test
public void testEmptyOptionalVariables() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a",
"applicationName = b", "AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 123",
"initialPositionInStream = TriM_Horizon", "maxGetRecordsThreadPool = 1" }, '\n'));
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = ABCD," + credentialName1,
"workerId = 123",
"initialPositionInStream = TriM_Horizon",
"maxGetRecordsThreadPool = 1"
},
'\n'));
assertThat(config.getMaxGetRecordsThreadPool(), equalTo(1));
assertThat(config.getRetryGetRecordsInSeconds(), nullValue());
}
@Test
public void testWithZeroValue() {
String test = StringUtils.join(new String[] { "streamName = a", "applicationName = b",
"AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 123",
"initialPositionInStream = TriM_Horizon", "maxGetRecordsThreadPool = 0",
"retryGetRecordsInSeconds = 0" }, '\n');
String test = StringUtils.join(
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = ABCD," + credentialName1,
"workerId = 123",
"initialPositionInStream = TriM_Horizon",
"maxGetRecordsThreadPool = 0",
"retryGetRecordsInSeconds = 0"
},
'\n');
getConfiguration(test);
}
@Test
public void testWithInvalidIntValue() {
String test = StringUtils.join(new String[] { "streamName = a", "applicationName = b",
"AWSCredentialsProvider = " + credentialName1, "workerId = 123", "failoverTimeMillis = 100nf" }, '\n');
String test = StringUtils.join(
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = " + credentialName1,
"workerId = 123",
"failoverTimeMillis = 100nf"
},
'\n');
getConfiguration(test);
}
@Test
public void testWithNegativeIntValue() {
String test = StringUtils.join(new String[] { "streamName = a", "applicationName = b",
"AWSCredentialsProvider = " + credentialName1, "workerId = 123", "failoverTimeMillis = -12" }, '\n');
String test = StringUtils.join(
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = " + credentialName1,
"workerId = 123",
"failoverTimeMillis = -12"
},
'\n');
// separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement
getConfiguration(test);
@ -252,8 +360,15 @@ public class KinesisClientLibConfiguratorTest {
@Test(expected = IllegalArgumentException.class)
public void testWithMissingCredentialsProvider() {
String test = StringUtils.join(new String[] { "streamName = a", "applicationName = b", "workerId = 123",
"failoverTimeMillis = 100", "shardSyncIntervalMillis = 500" }, '\n');
String test = StringUtils.join(
new String[] {
"streamName = a",
"applicationName = b",
"workerId = 123",
"failoverTimeMillis = 100",
"shardSyncIntervalMillis = 500"
},
'\n');
// separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement
getConfiguration(test);
@ -262,8 +377,13 @@ public class KinesisClientLibConfiguratorTest {
@Test
public void testWithMissingWorkerId() {
String test = StringUtils.join(
new String[] { "streamName = a", "applicationName = b", "AWSCredentialsProvider = " + credentialName1,
"failoverTimeMillis = 100", "shardSyncIntervalMillis = 500" },
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = " + credentialName1,
"failoverTimeMillis = 100",
"shardSyncIntervalMillis = 500"
},
'\n');
MultiLangDaemonConfiguration config = getConfiguration(test);
@ -274,46 +394,63 @@ public class KinesisClientLibConfiguratorTest {
@Test(expected = NullPointerException.class)
public void testWithMissingStreamNameAndMissingStreamArn() {
String test = StringUtils.join(new String[] {
"applicationName = b",
"AWSCredentialsProvider = " + credentialName1,
"workerId = 123",
"failoverTimeMillis = 100" },
String test = StringUtils.join(
new String[] {
"applicationName = b",
"AWSCredentialsProvider = " + credentialName1,
"workerId = 123",
"failoverTimeMillis = 100"
},
'\n');
getConfiguration(test);
}
@Test(expected = IllegalArgumentException.class)
public void testWithEmptyStreamNameAndMissingStreamArn() {
String test = StringUtils.join(new String[] {
"applicationName = b",
"AWSCredentialsProvider = " + credentialName1,
"workerId = 123",
"failoverTimeMillis = 100",
"streamName = ",
"streamArn = "},
String test = StringUtils.join(
new String[] {
"applicationName = b",
"AWSCredentialsProvider = " + credentialName1,
"workerId = 123",
"failoverTimeMillis = 100",
"streamName = ",
"streamArn = "
},
'\n');
getConfiguration(test);
}
@Test(expected = NullPointerException.class)
public void testWithMissingApplicationName() {
String test = StringUtils.join(new String[] { "streamName = a", "AWSCredentialsProvider = " + credentialName1,
"workerId = 123", "failoverTimeMillis = 100" }, '\n');
String test = StringUtils.join(
new String[] {
"streamName = a",
"AWSCredentialsProvider = " + credentialName1,
"workerId = 123",
"failoverTimeMillis = 100"
},
'\n');
getConfiguration(test);
}
@Test
public void testWithAWSCredentialsFailed() {
String test = StringUtils.join(
new String[] { "streamName = a", "applicationName = b", "AWSCredentialsProvider = " + credentialName2,
"failoverTimeMillis = 100", "shardSyncIntervalMillis = 500" },
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = " + credentialName2,
"failoverTimeMillis = 100",
"shardSyncIntervalMillis = 500"
},
'\n');
MultiLangDaemonConfiguration config = getConfiguration(test);
// separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement
try {
config.getKinesisCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials();
config.getKinesisCredentialsProvider()
.build(AwsCredentialsProvider.class)
.resolveCredentials();
fail("expect failure with wrong credentials provider");
} catch (Exception e) {
// succeed
@ -323,39 +460,63 @@ public class KinesisClientLibConfiguratorTest {
// TODO: fix this test
@Test
public void testWithDifferentAWSCredentialsForDynamoDBAndCloudWatch() {
String test = StringUtils.join(new String[] { "streamName = a", "applicationName = b",
"AWSCredentialsProvider = " + credentialNameKinesis,
"AWSCredentialsProviderDynamoDB = " + credentialNameDynamoDB,
"AWSCredentialsProviderCloudWatch = " + credentialNameCloudWatch, "failoverTimeMillis = 100",
"shardSyncIntervalMillis = 500" }, '\n');
String test = StringUtils.join(
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = " + credentialNameKinesis,
"AWSCredentialsProviderDynamoDB = " + credentialNameDynamoDB,
"AWSCredentialsProviderCloudWatch = " + credentialNameCloudWatch,
"failoverTimeMillis = 100",
"shardSyncIntervalMillis = 500"
},
'\n');
// separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement
final MultiLangDaemonConfiguration config = getConfiguration(test);
config.getKinesisCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials();
config.getDynamoDBCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials();
config.getCloudWatchCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials();
config.getKinesisCredentialsProvider()
.build(AwsCredentialsProvider.class)
.resolveCredentials();
config.getDynamoDBCredentialsProvider()
.build(AwsCredentialsProvider.class)
.resolveCredentials();
config.getCloudWatchCredentialsProvider()
.build(AwsCredentialsProvider.class)
.resolveCredentials();
}
// TODO: fix this test
@Test
public void testWithDifferentAWSCredentialsForDynamoDBAndCloudWatchFailed() {
String test = StringUtils.join(new String[] { "streamName = a", "applicationName = b",
"AWSCredentialsProvider = " + credentialNameKinesis,
"AWSCredentialsProviderDynamoDB = " + credentialName2,
"AWSCredentialsProviderCloudWatch = " + credentialName2, "failoverTimeMillis = 100",
"shardSyncIntervalMillis = 500" }, '\n');
String test = StringUtils.join(
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = " + credentialNameKinesis,
"AWSCredentialsProviderDynamoDB = " + credentialName2,
"AWSCredentialsProviderCloudWatch = " + credentialName2,
"failoverTimeMillis = 100",
"shardSyncIntervalMillis = 500"
},
'\n');
// separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement
final MultiLangDaemonConfiguration config = getConfiguration(test);
config.getKinesisCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials();
config.getKinesisCredentialsProvider()
.build(AwsCredentialsProvider.class)
.resolveCredentials();
try {
config.getDynamoDBCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials();
config.getDynamoDBCredentialsProvider()
.build(AwsCredentialsProvider.class)
.resolveCredentials();
fail("DynamoDB credential providers should fail.");
} catch (Exception e) {
// succeed
}
try {
config.getCloudWatchCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials();
config.getCloudWatchCredentialsProvider()
.build(AwsCredentialsProvider.class)
.resolveCredentials();
fail("CloudWatch credential providers should fail.");
} catch (Exception e) {
// succeed
@ -373,9 +534,7 @@ public class KinesisClientLibConfiguratorTest {
}
@Override
public void refresh() {
}
public void refresh() {}
}
/**
@ -389,9 +548,7 @@ public class KinesisClientLibConfiguratorTest {
}
@Override
public void refresh() {
}
public void refresh() {}
}
/**
@ -405,9 +562,7 @@ public class KinesisClientLibConfiguratorTest {
}
@Override
public void refresh() {
}
public void refresh() {}
}
/**
@ -421,9 +576,7 @@ public class KinesisClientLibConfiguratorTest {
}
@Override
public void refresh() {
}
public void refresh() {}
}
/**
@ -437,9 +590,7 @@ public class KinesisClientLibConfiguratorTest {
}
@Override
public void refresh() {
}
public void refresh() {}
}
private MultiLangDaemonConfiguration getConfiguration(String configString) {

104
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/MultiLangDaemonConfigurationTest.java
View file

@ -15,12 +15,6 @@
package software.amazon.kinesis.multilang.config;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import org.apache.commons.beanutils.BeanUtilsBean;
import org.apache.commons.beanutils.ConvertUtilsBean;
import org.junit.After;
@ -31,12 +25,17 @@ import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
import software.amazon.kinesis.processor.ShardRecordProcessorFactory;
import software.amazon.kinesis.retrieval.fanout.FanOutConfig;
import software.amazon.kinesis.retrieval.polling.PollingConfig;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
@RunWith(MockitoJUnitRunner.class)
public class MultiLangDaemonConfigurationTest {
@ -69,7 +68,6 @@ public class MultiLangDaemonConfigurationTest {
}
}
public MultiLangDaemonConfiguration baseConfiguration() {
MultiLangDaemonConfiguration configuration = new MultiLangDaemonConfiguration(utilsBean, convertUtilsBean);
configuration.setApplicationName("Test");
@ -84,8 +82,8 @@ public class MultiLangDaemonConfigurationTest {
MultiLangDaemonConfiguration configuration = baseConfiguration();
configuration.setMaxLeasesForWorker(10);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration
.resolvedConfiguration(shardRecordProcessorFactory);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
configuration.resolvedConfiguration(shardRecordProcessorFactory);
assertThat(resolvedConfiguration.leaseManagementConfig.maxLeasesForWorker(), equalTo(10));
}
@ -95,8 +93,8 @@ public class MultiLangDaemonConfigurationTest {
MultiLangDaemonConfiguration configuration = baseConfiguration();
configuration.setEnablePriorityLeaseAssignment(false);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration.resolvedConfiguration(
shardRecordProcessorFactory);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
configuration.resolvedConfiguration(shardRecordProcessorFactory);
assertThat(resolvedConfiguration.leaseManagementConfig.enablePriorityLeaseAssignment(), equalTo(false));
}
@ -105,11 +103,11 @@ public class MultiLangDaemonConfigurationTest {
public void testDefaultRetrievalConfig() {
MultiLangDaemonConfiguration configuration = baseConfiguration();
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration
.resolvedConfiguration(shardRecordProcessorFactory);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
configuration.resolvedConfiguration(shardRecordProcessorFactory);
assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(),
instanceOf(FanOutConfig.class));
assertThat(
resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(FanOutConfig.class));
}
@Test
@ -118,18 +116,20 @@ public class MultiLangDaemonConfigurationTest {
configuration.setMaxRecords(10);
configuration.setIdleTimeBetweenReadsInMillis(60000);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration
.resolvedConfiguration(shardRecordProcessorFactory);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
configuration.resolvedConfiguration(shardRecordProcessorFactory);
assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(),
instanceOf(PollingConfig.class));
assertEquals(10,
((PollingConfig) resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig()).maxRecords());
assertEquals(60000,
((PollingConfig) resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig())
.idleTimeBetweenReadsInMillis());
assertThat(
resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(PollingConfig.class));
assertEquals(
10,
((PollingConfig) resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig()).maxRecords());
assertEquals(
60000,
((PollingConfig) resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig())
.idleTimeBetweenReadsInMillis());
assertTrue(((PollingConfig) resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig())
.usePollingConfigIdleTimeValue());
.usePollingConfigIdleTimeValue());
}
@Test
@ -137,11 +137,11 @@ public class MultiLangDaemonConfigurationTest {
MultiLangDaemonConfiguration configuration = baseConfiguration();
configuration.setRetrievalMode(RetrievalMode.FANOUT);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration
.resolvedConfiguration(shardRecordProcessorFactory);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
configuration.resolvedConfiguration(shardRecordProcessorFactory);
assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(),
instanceOf(FanOutConfig.class));
assertThat(
resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(FanOutConfig.class));
}
@Test
@ -149,37 +149,39 @@ public class MultiLangDaemonConfigurationTest {
MultiLangDaemonConfiguration configuration = baseConfiguration();
configuration.setRetrievalMode(RetrievalMode.POLLING);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration
.resolvedConfiguration(shardRecordProcessorFactory);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
configuration.resolvedConfiguration(shardRecordProcessorFactory);
assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(),
instanceOf(PollingConfig.class));
assertThat(
resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(PollingConfig.class));
}
@Test
public void testRetrievalModeSetForPollingString() throws Exception {
MultiLangDaemonConfiguration configuration = baseConfiguration();
utilsBean.setProperty(configuration, "retrievalMode", RetrievalMode.POLLING.name().toLowerCase());
utilsBean.setProperty(
configuration, "retrievalMode", RetrievalMode.POLLING.name().toLowerCase());
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration
.resolvedConfiguration(shardRecordProcessorFactory);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
configuration.resolvedConfiguration(shardRecordProcessorFactory);
assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(),
instanceOf(PollingConfig.class));
assertThat(
resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(PollingConfig.class));
}
@Test
public void testRetrievalModeSetForFanoutString() throws Exception {
MultiLangDaemonConfiguration configuration = baseConfiguration();
utilsBean.setProperty(configuration, "retrievalMode", RetrievalMode.FANOUT.name().toLowerCase());
utilsBean.setProperty(
configuration, "retrievalMode", RetrievalMode.FANOUT.name().toLowerCase());
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration
.resolvedConfiguration(shardRecordProcessorFactory);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
configuration.resolvedConfiguration(shardRecordProcessorFactory);
assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(),
instanceOf(FanOutConfig.class));
assertThat(
resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(FanOutConfig.class));
}
@Test
@ -196,7 +198,7 @@ public class MultiLangDaemonConfigurationTest {
// TODO : Enable this test once https://github.com/awslabs/amazon-kinesis-client/issues/692 is resolved
public void testmetricsEnabledDimensions() {
MultiLangDaemonConfiguration configuration = baseConfiguration();
configuration.setMetricsEnabledDimensions(new String[]{"Operation"});
configuration.setMetricsEnabledDimensions(new String[] {"Operation"});
configuration.resolvedConfiguration(shardRecordProcessorFactory);
}
@ -209,14 +211,14 @@ public class MultiLangDaemonConfigurationTest {
configuration.setRetrievalMode(RetrievalMode.FANOUT);
configuration.getFanoutConfig().setConsumerArn(consumerArn);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration
.resolvedConfiguration(shardRecordProcessorFactory);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
configuration.resolvedConfiguration(shardRecordProcessorFactory);
assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(),
instanceOf(FanOutConfig.class));
FanOutConfig fanOutConfig = (FanOutConfig) resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig();
assertThat(
resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(FanOutConfig.class));
FanOutConfig fanOutConfig =
(FanOutConfig) resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig();
assertThat(fanOutConfig.consumerArn(), equalTo(consumerArn));
}
}

20
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/PollingConfigBeanTest.java
View file

@ -15,6 +15,8 @@
package software.amazon.kinesis.multilang.config;
import java.util.Optional;
import org.apache.commons.beanutils.BeanUtilsBean;
import org.apache.commons.beanutils.ConvertUtilsBean;
import org.junit.Test;
@ -24,8 +26,6 @@ import org.mockito.runners.MockitoJUnitRunner;
import software.amazon.awssdk.services.kinesis.KinesisAsyncClient;
import software.amazon.kinesis.retrieval.polling.PollingConfig;
import java.util.Optional;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.junit.Assert.assertThat;
@ -46,17 +46,23 @@ public class PollingConfigBeanTest {
ConvertUtilsBean convertUtilsBean = new ConvertUtilsBean();
BeanUtilsBean utilsBean = new BeanUtilsBean(convertUtilsBean);
MultiLangDaemonConfiguration multiLangDaemonConfiguration = new MultiLangDaemonConfiguration(utilsBean, convertUtilsBean);
MultiLangDaemonConfiguration multiLangDaemonConfiguration =
new MultiLangDaemonConfiguration(utilsBean, convertUtilsBean);
multiLangDaemonConfiguration.setStreamName("test-stream");
PollingConfig pollingConfig = pollingConfigBean.build(kinesisAsyncClient, multiLangDaemonConfiguration);
assertThat(pollingConfig.kinesisClient(), equalTo(kinesisAsyncClient));
assertThat(pollingConfig.streamName(), equalTo(multiLangDaemonConfiguration.getStreamName()));
assertThat(pollingConfig.idleTimeBetweenReadsInMillis(), equalTo(pollingConfigBean.getIdleTimeBetweenReadsInMillis()));
assertThat(pollingConfig.maxGetRecordsThreadPool(), equalTo(Optional.of(pollingConfigBean.getMaxGetRecordsThreadPool())));
assertThat(
pollingConfig.idleTimeBetweenReadsInMillis(),
equalTo(pollingConfigBean.getIdleTimeBetweenReadsInMillis()));
assertThat(
pollingConfig.maxGetRecordsThreadPool(),
equalTo(Optional.of(pollingConfigBean.getMaxGetRecordsThreadPool())));
assertThat(pollingConfig.maxRecords(), equalTo(pollingConfigBean.getMaxRecords()));
assertThat(pollingConfig.retryGetRecordsInSeconds(), equalTo(Optional.of(pollingConfigBean.getRetryGetRecordsInSeconds())));
assertThat(
pollingConfig.retryGetRecordsInSeconds(),
equalTo(Optional.of(pollingConfigBean.getRetryGetRecordsInSeconds())));
}
}

48
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/messages/JsonFriendlyRecordTest.java
View file

@ -15,11 +15,6 @@
package software.amazon.kinesis.multilang.messages;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.nullValue;
import static org.hamcrest.CoreMatchers.sameInstance;
import static org.junit.Assert.assertThat;
import java.nio.ByteBuffer;
import java.time.Instant;
import java.util.Arrays;
@ -31,9 +26,13 @@ import org.hamcrest.Description;
import org.hamcrest.Matcher;
import org.hamcrest.TypeSafeDiagnosingMatcher;
import org.junit.Test;
import software.amazon.kinesis.retrieval.KinesisClientRecord;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.nullValue;
import static org.hamcrest.CoreMatchers.sameInstance;
import static org.junit.Assert.assertThat;
public class JsonFriendlyRecordTest {
private KinesisClientRecord kinesisClientRecord;
@ -48,7 +47,7 @@ public class JsonFriendlyRecordTest {
@Test
public void testRecordHandlesNoByteArrayBuffer() {
byte[] expected = new byte[] { 1, 2, 3, 4 };
byte[] expected = new byte[] {1, 2, 3, 4};
ByteBuffer expectedBuffer = ByteBuffer.allocateDirect(expected.length);
@ -64,7 +63,7 @@ public class JsonFriendlyRecordTest {
@Test
public void testRecordHandlesArrayByteBuffer() {
ByteBuffer expected = ByteBuffer.wrap(new byte[] { 1, 2, 3, 4 });
ByteBuffer expected = ByteBuffer.wrap(new byte[] {1, 2, 3, 4});
kinesisClientRecord = defaultRecord().data(expected).build();
JsonFriendlyRecord jsonFriendlyRecord = JsonFriendlyRecord.fromKinesisClientRecord(kinesisClientRecord);
@ -82,14 +81,15 @@ public class JsonFriendlyRecordTest {
private RecordMatcher(KinesisClientRecord expected) {
this.matchers = Arrays.asList(
new FieldMatcher<>("approximateArrivalTimestamp",
new FieldMatcher<>(
"approximateArrivalTimestamp",
equalTo(expected.approximateArrivalTimestamp().toEpochMilli()),
JsonFriendlyRecord::getApproximateArrivalTimestamp),
new FieldMatcher<>("partitionKey", expected::partitionKey, JsonFriendlyRecord::getPartitionKey),
new FieldMatcher<>("sequenceNumber", expected::sequenceNumber,
JsonFriendlyRecord::getSequenceNumber),
new FieldMatcher<>("subSequenceNumber", expected::subSequenceNumber,
JsonFriendlyRecord::getSubSequenceNumber),
new FieldMatcher<>(
"sequenceNumber", expected::sequenceNumber, JsonFriendlyRecord::getSequenceNumber),
new FieldMatcher<>(
"subSequenceNumber", expected::subSequenceNumber, JsonFriendlyRecord::getSubSequenceNumber),
new FieldMatcher<>("data", dataEquivalentTo(expected.data()), JsonFriendlyRecord::getData));
this.expected = expected;
@ -97,13 +97,16 @@ public class JsonFriendlyRecordTest {
@Override
protected boolean matchesSafely(JsonFriendlyRecord item, Description mismatchDescription) {
return matchers.stream().map(m -> {
if (!m.matches(item)) {
m.describeMismatch(item, mismatchDescription);
return false;
}
return true;
}).reduce((l, r) -> l && r).orElse(true);
return matchers.stream()
.map(m -> {
if (!m.matches(item)) {
m.describeMismatch(item, mismatchDescription);
return false;
}
return true;
})
.reduce((l, r) -> l && r)
.orElse(true);
}
@Override
@ -160,8 +163,9 @@ public class JsonFriendlyRecordTest {
}
private KinesisClientRecord.KinesisClientRecordBuilder defaultRecord() {
return KinesisClientRecord.builder().partitionKey("test-partition").sequenceNumber("123")
return KinesisClientRecord.builder()
.partitionKey("test-partition")
.sequenceNumber("123")
.approximateArrivalTimestamp(Instant.now());
}
}

59
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/messages/MessageTest.java
View file

@ -17,57 +17,57 @@ package software.amazon.kinesis.multilang.messages;
import java.nio.ByteBuffer;
import java.util.Collections;
import org.junit.Assert;
import org.junit.Test;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.Assert;
import org.junit.Test;
import software.amazon.kinesis.lifecycle.ShutdownReason;
import software.amazon.kinesis.lifecycle.events.InitializationInput;
import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput;
import software.amazon.kinesis.lifecycle.ShutdownReason;
import software.amazon.kinesis.retrieval.KinesisClientRecord;
public class MessageTest {
@Test
public void toStringTest() {
Message[] messages = new Message[]{
new CheckpointMessage("1234567890", 0L, null),
new InitializeMessage(InitializationInput.builder().shardId("shard-123").build()),
new ProcessRecordsMessage(ProcessRecordsInput.builder()
.records(Collections.singletonList(
KinesisClientRecord.builder()
.data(ByteBuffer.wrap("cat".getBytes()))
.partitionKey("cat")
.sequenceNumber("555")
.build()))
.build()),
new ShutdownMessage(ShutdownReason.LEASE_LOST),
new StatusMessage("processRecords"),
new InitializeMessage(),
new ProcessRecordsMessage(),
new ShutdownRequestedMessage(),
new LeaseLostMessage(),
new ShardEndedMessage(),
Message[] messages = new Message[] {
new CheckpointMessage("1234567890", 0L, null),
new InitializeMessage(
InitializationInput.builder().shardId("shard-123").build()),
new ProcessRecordsMessage(ProcessRecordsInput.builder()
.records(Collections.singletonList(KinesisClientRecord.builder()
.data(ByteBuffer.wrap("cat".getBytes()))
.partitionKey("cat")
.sequenceNumber("555")
.build()))
.build()),
new ShutdownMessage(ShutdownReason.LEASE_LOST),
new StatusMessage("processRecords"),
new InitializeMessage(),
new ProcessRecordsMessage(),
new ShutdownRequestedMessage(),
new LeaseLostMessage(),
new ShardEndedMessage(),
};
// TODO: fix this
// TODO: fix this
for (int i = 0; i < messages.length; i++) {
System.out.println(messages[i].toString());
Assert.assertTrue("Each message should contain the action field", messages[i].toString().contains("action"));
Assert.assertTrue(
"Each message should contain the action field",
messages[i].toString().contains("action"));
}
// Hit this constructor
KinesisClientRecord defaultJsonFriendlyRecord = KinesisClientRecord.builder().build();
KinesisClientRecord defaultJsonFriendlyRecord =
KinesisClientRecord.builder().build();
Assert.assertNull(defaultJsonFriendlyRecord.partitionKey());
Assert.assertNull(defaultJsonFriendlyRecord.data());
Assert.assertNull(defaultJsonFriendlyRecord.sequenceNumber());
Assert.assertNull(new ShutdownMessage(null).getReason());
// Hit the bad object mapping path
Message withBadMapper = new Message() {
}.withObjectMapper(new ObjectMapper() {
Message withBadMapper = new Message() {}.withObjectMapper(new ObjectMapper() {
/**
*
*/
@ -75,8 +75,7 @@ public class MessageTest {
@Override
public String writeValueAsString(Object m) throws JsonProcessingException {
throw new JsonProcessingException(new Throwable()) {
};
throw new JsonProcessingException(new Throwable()) {};
}
});
String s = withBadMapper.toString();

3
amazon-kinesis-client/src/main/java/software/amazon/kinesis/annotations/KinesisClientInternalApi.java
View file

@ -22,5 +22,4 @@ import java.lang.annotation.RetentionPolicy;
* Any class/method/variable marked with this annotation is subject to breaking changes between minor releases.
*/
@Retention(RetentionPolicy.CLASS)
public @interface KinesisClientInternalApi {
}
public @interface KinesisClientInternalApi {}

5
amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/Checkpoint.java
View file

@ -40,7 +40,10 @@ public class Checkpoint {
* @param pendingCheckpoint the pending checkpoint sequence number - can be null.
* @param pendingCheckpointState the pending checkpoint state - can be null.
*/
public Checkpoint(final ExtendedSequenceNumber checkpoint, final ExtendedSequenceNumber pendingCheckpoint, byte[] pendingCheckpointState) {
public Checkpoint(
final ExtendedSequenceNumber checkpoint,
final ExtendedSequenceNumber pendingCheckpoint,
byte[] pendingCheckpointState) {
if (checkpoint == null || checkpoint.sequenceNumber().isEmpty()) {
throw new IllegalArgumentException("Checkpoint cannot be null or empty");
}

1
amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/CheckpointConfig.java
View file

@ -15,7 +15,6 @@
package software.amazon.kinesis.checkpoint;
import lombok.Data;
import lombok.experimental.Accessors;
import software.amazon.kinesis.checkpoint.dynamodb.DynamoDBCheckpointFactory;

4
amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/DoesNothingPreparedCheckpointer.java
View file

@ -60,9 +60,7 @@ public class DoesNothingPreparedCheckpointer implements PreparedCheckpointer {
@Override
public void checkpoint()
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
IllegalArgumentException {
IllegalArgumentException {
// This method does nothing
}
}

14
amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/SequenceNumberValidator.java
View file

@ -20,10 +20,9 @@ import java.util.Collections;
import java.util.List;
import java.util.Optional;
import org.apache.commons.lang3.StringUtils;
import lombok.Data;
import lombok.experimental.Accessors;
import org.apache.commons.lang3.StringUtils;
/**
* This supports extracting the shardId from a sequence number.
@ -98,11 +97,15 @@ public class SequenceNumberValidator {
}
}
private static final List<SequenceNumberReader> SEQUENCE_NUMBER_READERS = Collections
.singletonList(new V2SequenceNumberReader());
private static final List<SequenceNumberReader> SEQUENCE_NUMBER_READERS =
Collections.singletonList(new V2SequenceNumberReader());
private Optional<SequenceNumberComponents> retrieveComponentsFor(String sequenceNumber) {
return SEQUENCE_NUMBER_READERS.stream().map(r -> r.read(sequenceNumber)).filter(Optional::isPresent).map(Optional::get).findFirst();
return SEQUENCE_NUMBER_READERS.stream()
.map(r -> r.read(sequenceNumber))
.filter(Optional::isPresent)
.map(Optional::get)
.findFirst();
}
/**
@ -184,5 +187,4 @@ public class SequenceNumberValidator {
public Optional<Boolean> validateSequenceNumberForShard(String sequenceNumber, String shardId) {
return shardIdFor(sequenceNumber).map(s -> StringUtils.equalsIgnoreCase(s, shardId));
}
}

10
amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/ShardPreparedCheckpointer.java
View file

@ -38,8 +38,8 @@ public class ShardPreparedCheckpointer implements PreparedCheckpointer {
* @param pendingCheckpointSequenceNumber sequence number to checkpoint at
* @param checkpointer checkpointer to use
*/
public ShardPreparedCheckpointer(ExtendedSequenceNumber pendingCheckpointSequenceNumber,
RecordProcessorCheckpointer checkpointer) {
public ShardPreparedCheckpointer(
ExtendedSequenceNumber pendingCheckpointSequenceNumber, RecordProcessorCheckpointer checkpointer) {
this.pendingCheckpointSequenceNumber = pendingCheckpointSequenceNumber;
this.checkpointer = checkpointer;
}
@ -58,8 +58,8 @@ public class ShardPreparedCheckpointer implements PreparedCheckpointer {
@Override
public void checkpoint()
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
IllegalArgumentException {
checkpointer.checkpoint(pendingCheckpointSequenceNumber.sequenceNumber(),
pendingCheckpointSequenceNumber.subSequenceNumber());
IllegalArgumentException {
checkpointer.checkpoint(
pendingCheckpointSequenceNumber.sequenceNumber(), pendingCheckpointSequenceNumber.subSequenceNumber());
}
}

106
amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/ShardRecordProcessorCheckpointer.java
View file

@ -41,16 +41,22 @@ import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber;
public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpointer {
@NonNull
private final ShardInfo shardInfo;
@NonNull
@Getter @Accessors(fluent = true)
@Getter
@Accessors(fluent = true)
private final Checkpointer checkpointer;
// Set to the last value set via checkpoint().
// Sample use: verify application shutdown() invoked checkpoint() at the end of a shard.
@Getter @Accessors(fluent = true)
@Getter
@Accessors(fluent = true)
private ExtendedSequenceNumber lastCheckpointValue;
@Getter @Accessors(fluent = true)
@Getter
@Accessors(fluent = true)
private ExtendedSequenceNumber largestPermittedCheckpointValue;
private ExtendedSequenceNumber sequenceNumberAtShardEnd;
/**
@ -60,8 +66,11 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
public synchronized void checkpoint()
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
if (log.isDebugEnabled()) {
log.debug("Checkpointing {}, token {} at largest permitted value {}", ShardInfo.getLeaseKey(shardInfo),
shardInfo.concurrencyToken(), this.largestPermittedCheckpointValue);
log.debug(
"Checkpointing {}, token {} at largest permitted value {}",
ShardInfo.getLeaseKey(shardInfo),
shardInfo.concurrencyToken(),
this.largestPermittedCheckpointValue);
}
advancePosition(this.largestPermittedCheckpointValue);
}
@ -71,15 +80,15 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
*/
@Override
public synchronized void checkpoint(Record record)
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
IllegalArgumentException {
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
IllegalArgumentException {
// TODO: UserRecord Deprecation
if (record == null) {
throw new IllegalArgumentException("Could not checkpoint a null record");
} /* else if (record instanceof UserRecord) {
checkpoint(record.sequenceNumber(), ((UserRecord) record).subSequenceNumber());
} */ else {
checkpoint(record.sequenceNumber(), ((UserRecord) record).subSequenceNumber());
} */ else {
checkpoint(record.sequenceNumber(), 0);
}
}
@ -89,8 +98,8 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
*/
@Override
public synchronized void checkpoint(String sequenceNumber)
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
IllegalArgumentException {
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
IllegalArgumentException {
checkpoint(sequenceNumber, 0);
}
@ -99,12 +108,12 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
*/
@Override
public synchronized void checkpoint(String sequenceNumber, long subSequenceNumber)
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
IllegalArgumentException {
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
IllegalArgumentException {
if (subSequenceNumber < 0) {
throw new IllegalArgumentException("Could not checkpoint at invalid, negative subsequence number "
+ subSequenceNumber);
throw new IllegalArgumentException(
"Could not checkpoint at invalid, negative subsequence number " + subSequenceNumber);
}
/*
@ -116,15 +125,18 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
&& newCheckpoint.compareTo(largestPermittedCheckpointValue) <= 0) {
if (log.isDebugEnabled()) {
log.debug("Checkpointing {}, token {} at specific extended sequence number {}", ShardInfo.getLeaseKey(shardInfo),
shardInfo.concurrencyToken(), newCheckpoint);
log.debug(
"Checkpointing {}, token {} at specific extended sequence number {}",
ShardInfo.getLeaseKey(shardInfo),
shardInfo.concurrencyToken(),
newCheckpoint);
}
this.advancePosition(newCheckpoint);
} else {
throw new IllegalArgumentException(String.format(
"Could not checkpoint at extended sequence number %s as it did not fall into acceptable range "
+ "between the last checkpoint %s and the greatest extended sequence number passed to this "
+ "record processor %s",
+ "between the last checkpoint %s and the greatest extended sequence number passed to this "
+ "record processor %s",
newCheckpoint, this.lastCheckpointValue, this.largestPermittedCheckpointValue));
}
}
@ -161,8 +173,8 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
if (record == null) {
throw new IllegalArgumentException("Could not prepare checkpoint a null record");
} /*else if (record instanceof UserRecord) {
return prepareCheckpoint(record.sequenceNumber(), ((UserRecord) record).subSequenceNumber());
} */ else {
return prepareCheckpoint(record.sequenceNumber(), ((UserRecord) record).subSequenceNumber());
} */ else {
return prepareCheckpoint(record.sequenceNumber(), 0, applicationState);
}
}
@ -190,7 +202,8 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
*/
@Override
public PreparedCheckpointer prepareCheckpoint(String sequenceNumber, byte[] applicationState)
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException, IllegalArgumentException {
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
IllegalArgumentException {
return prepareCheckpoint(sequenceNumber, 0, applicationState);
}
@ -207,11 +220,13 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
* {@inheritDoc}
*/
@Override
public PreparedCheckpointer prepareCheckpoint(String sequenceNumber, long subSequenceNumber, byte[] applicationState)
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException, IllegalArgumentException {
public PreparedCheckpointer prepareCheckpoint(
String sequenceNumber, long subSequenceNumber, byte[] applicationState)
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
IllegalArgumentException {
if (subSequenceNumber < 0) {
throw new IllegalArgumentException("Could not checkpoint at invalid, negative subsequence number "
+ subSequenceNumber);
throw new IllegalArgumentException(
"Could not checkpoint at invalid, negative subsequence number " + subSequenceNumber);
}
/*
@ -223,8 +238,11 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
&& pendingCheckpoint.compareTo(largestPermittedCheckpointValue) <= 0) {
if (log.isDebugEnabled()) {
log.debug("Preparing checkpoint {}, token {} at specific extended sequence number {}",
ShardInfo.getLeaseKey(shardInfo), shardInfo.concurrencyToken(), pendingCheckpoint);
log.debug(
"Preparing checkpoint {}, token {} at specific extended sequence number {}",
ShardInfo.getLeaseKey(shardInfo),
shardInfo.concurrencyToken(),
pendingCheckpoint);
}
return doPrepareCheckpoint(pendingCheckpoint, applicationState);
} else {
@ -258,7 +276,6 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
this.sequenceNumberAtShardEnd = extendedSequenceNumber;
}
/**
* Internal API - has package level access only for testing purposes.
*
@ -270,12 +287,12 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
* @throws InvalidStateException
*/
void advancePosition(String sequenceNumber)
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
advancePosition(new ExtendedSequenceNumber(sequenceNumber));
}
void advancePosition(ExtendedSequenceNumber extendedSequenceNumber)
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
ExtendedSequenceNumber checkpointToRecord = extendedSequenceNumber;
if (sequenceNumberAtShardEnd != null && sequenceNumberAtShardEnd.equals(extendedSequenceNumber)) {
// If we are about to checkpoint the very last sequence number for this shard, we might as well
@ -287,12 +304,18 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
if (extendedSequenceNumber != null && !extendedSequenceNumber.equals(lastCheckpointValue)) {
try {
if (log.isDebugEnabled()) {
log.debug("Setting {}, token {} checkpoint to {}", ShardInfo.getLeaseKey(shardInfo),
shardInfo.concurrencyToken(), checkpointToRecord);
log.debug(
"Setting {}, token {} checkpoint to {}",
ShardInfo.getLeaseKey(shardInfo),
shardInfo.concurrencyToken(),
checkpointToRecord);
}
checkpointer.setCheckpoint(ShardInfo.getLeaseKey(shardInfo), checkpointToRecord, shardInfo.concurrencyToken());
checkpointer.setCheckpoint(
ShardInfo.getLeaseKey(shardInfo), checkpointToRecord, shardInfo.concurrencyToken());
lastCheckpointValue = checkpointToRecord;
} catch (ThrottlingException | ShutdownException | InvalidStateException
} catch (ThrottlingException
| ShutdownException
| InvalidStateException
| KinesisClientLibDependencyException e) {
throw e;
} catch (KinesisClientLibException e) {
@ -325,7 +348,8 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
* @throws ThrottlingException
* @throws ShutdownException
*/
private PreparedCheckpointer doPrepareCheckpoint(ExtendedSequenceNumber extendedSequenceNumber, byte[] applicationState)
private PreparedCheckpointer doPrepareCheckpoint(
ExtendedSequenceNumber extendedSequenceNumber, byte[] applicationState)
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
ExtendedSequenceNumber newPrepareCheckpoint = extendedSequenceNumber;
@ -343,8 +367,14 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
}
try {
checkpointer.prepareCheckpoint(ShardInfo.getLeaseKey(shardInfo), newPrepareCheckpoint, shardInfo.concurrencyToken(), applicationState);
} catch (ThrottlingException | ShutdownException | InvalidStateException
checkpointer.prepareCheckpoint(
ShardInfo.getLeaseKey(shardInfo),
newPrepareCheckpoint,
shardInfo.concurrencyToken(),
applicationState);
} catch (ThrottlingException
| ShutdownException
| InvalidStateException
| KinesisClientLibDependencyException e) {
throw e;
} catch (KinesisClientLibException e) {

5
amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/dynamodb/DynamoDBCheckpointFactory.java
View file

@ -29,9 +29,8 @@ import software.amazon.kinesis.processor.Checkpointer;
@KinesisClientInternalApi
public class DynamoDBCheckpointFactory implements CheckpointFactory {
@Override
public Checkpointer createCheckpointer(final LeaseCoordinator leaseLeaseCoordinator,
final LeaseRefresher leaseRefresher) {
public Checkpointer createCheckpointer(
final LeaseCoordinator leaseLeaseCoordinator, final LeaseRefresher leaseRefresher) {
return new DynamoDBCheckpointer(leaseLeaseCoordinator, leaseRefresher);
}
}

42
amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/dynamodb/DynamoDBCheckpointer.java
View file

@ -19,7 +19,6 @@ import java.util.Objects;
import java.util.UUID;
import com.google.common.annotations.VisibleForTesting;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@ -48,14 +47,16 @@ import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber;
public class DynamoDBCheckpointer implements Checkpointer {
@NonNull
private final LeaseCoordinator leaseCoordinator;
@NonNull
private final LeaseRefresher leaseRefresher;
private String operation;
@Override
public void setCheckpoint(final String leaseKey, final ExtendedSequenceNumber checkpointValue,
final String concurrencyToken) throws KinesisClientLibException {
public void setCheckpoint(
final String leaseKey, final ExtendedSequenceNumber checkpointValue, final String concurrencyToken)
throws KinesisClientLibException {
try {
boolean wasSuccessful = setCheckpoint(leaseKey, checkpointValue, UUID.fromString(concurrencyToken));
if (!wasSuccessful) {
@ -97,17 +98,22 @@ public class DynamoDBCheckpointer implements Checkpointer {
}
@Override
public void prepareCheckpoint(final String leaseKey, final ExtendedSequenceNumber pendingCheckpoint,
final String concurrencyToken) throws KinesisClientLibException {
public void prepareCheckpoint(
final String leaseKey, final ExtendedSequenceNumber pendingCheckpoint, final String concurrencyToken)
throws KinesisClientLibException {
prepareCheckpoint(leaseKey, pendingCheckpoint, concurrencyToken, null);
}
@Override
public void prepareCheckpoint(String leaseKey, ExtendedSequenceNumber pendingCheckpoint, String concurrencyToken,
byte[] pendingCheckpointState) throws KinesisClientLibException {
public void prepareCheckpoint(
String leaseKey,
ExtendedSequenceNumber pendingCheckpoint,
String concurrencyToken,
byte[] pendingCheckpointState)
throws KinesisClientLibException {
try {
boolean wasSuccessful =
prepareCheckpoint(leaseKey, pendingCheckpoint, UUID.fromString(concurrencyToken), pendingCheckpointState);
boolean wasSuccessful = prepareCheckpoint(
leaseKey, pendingCheckpoint, UUID.fromString(concurrencyToken), pendingCheckpointState);
if (!wasSuccessful) {
throw new ShutdownException(
"Can't prepare checkpoint - instance doesn't hold the lease for this shard");
@ -128,8 +134,10 @@ public class DynamoDBCheckpointer implements Checkpointer {
throws DependencyException, InvalidStateException, ProvisionedThroughputException {
Lease lease = leaseCoordinator.getCurrentlyHeldLease(leaseKey);
if (lease == null) {
log.info("Worker {} could not update checkpoint for shard {} because it does not hold the lease",
leaseCoordinator.workerIdentifier(), leaseKey);
log.info(
"Worker {} could not update checkpoint for shard {} because it does not hold the lease",
leaseCoordinator.workerIdentifier(),
leaseKey);
return false;
}
@ -141,12 +149,18 @@ public class DynamoDBCheckpointer implements Checkpointer {
return leaseCoordinator.updateLease(lease, concurrencyToken, operation, leaseKey);
}
boolean prepareCheckpoint(String leaseKey, ExtendedSequenceNumber pendingCheckpoint, UUID concurrencyToken, byte[] pendingCheckpointState)
boolean prepareCheckpoint(
String leaseKey,
ExtendedSequenceNumber pendingCheckpoint,
UUID concurrencyToken,
byte[] pendingCheckpointState)
throws DependencyException, InvalidStateException, ProvisionedThroughputException {
Lease lease = leaseCoordinator.getCurrentlyHeldLease(leaseKey);
if (lease == null) {
log.info("Worker {} could not prepare checkpoint for shard {} because it does not hold the lease",
leaseCoordinator.workerIdentifier(), leaseKey);
log.info(
"Worker {} could not prepare checkpoint for shard {} because it does not hold the lease",
leaseCoordinator.workerIdentifier(),
leaseKey);
return false;
}

2
amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/CommonCalculations.java
View file

@ -15,10 +15,8 @@
package software.amazon.kinesis.common;
public class CommonCalculations {
/**
* Convenience method for calculating renewer intervals in milliseconds.
*

57
amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/ConfigsBuilder.java
View file

@ -19,12 +19,11 @@ import java.util.function.Function;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.NonNull;
import lombok.Setter;
import lombok.ToString;
import org.apache.commons.lang3.StringUtils;
import lombok.NonNull;
import lombok.experimental.Accessors;
import org.apache.commons.lang3.StringUtils;
import software.amazon.awssdk.arns.Arn;
import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient;
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
@ -35,9 +34,9 @@ import software.amazon.kinesis.coordinator.CoordinatorConfig;
import software.amazon.kinesis.leases.LeaseManagementConfig;
import software.amazon.kinesis.lifecycle.LifecycleConfig;
import software.amazon.kinesis.metrics.MetricsConfig;
import software.amazon.kinesis.processor.MultiStreamTracker;
import software.amazon.kinesis.processor.ProcessorConfig;
import software.amazon.kinesis.processor.ShardRecordProcessorFactory;
import software.amazon.kinesis.processor.MultiStreamTracker;
import software.amazon.kinesis.processor.SingleStreamTracker;
import software.amazon.kinesis.processor.StreamTracker;
import software.amazon.kinesis.retrieval.RetrievalConfig;
@ -45,7 +44,10 @@ import software.amazon.kinesis.retrieval.RetrievalConfig;
/**
* This Builder is useful to create all configurations for the KCL with default values.
*/
@Getter @Setter @ToString @EqualsAndHashCode
@Getter
@Setter
@ToString
@EqualsAndHashCode
@Accessors(fluent = true)
public class ConfigsBuilder {
/**
@ -139,11 +141,16 @@ public class ConfigsBuilder {
* @param workerIdentifier
* @param shardRecordProcessorFactory
*/
public ConfigsBuilder(@NonNull String streamName, @NonNull String applicationName,
@NonNull KinesisAsyncClient kinesisClient, @NonNull DynamoDbAsyncClient dynamoDBClient,
@NonNull CloudWatchAsyncClient cloudWatchClient, @NonNull String workerIdentifier,
public ConfigsBuilder(
@NonNull String streamName,
@NonNull String applicationName,
@NonNull KinesisAsyncClient kinesisClient,
@NonNull DynamoDbAsyncClient dynamoDBClient,
@NonNull CloudWatchAsyncClient cloudWatchClient,
@NonNull String workerIdentifier,
@NonNull ShardRecordProcessorFactory shardRecordProcessorFactory) {
this(new SingleStreamTracker(streamName),
this(
new SingleStreamTracker(streamName),
applicationName,
kinesisClient,
dynamoDBClient,
@ -163,11 +170,16 @@ public class ConfigsBuilder {
* @param workerIdentifier
* @param shardRecordProcessorFactory
*/
public ConfigsBuilder(@NonNull Arn streamArn, @NonNull String applicationName,
@NonNull KinesisAsyncClient kinesisClient, @NonNull DynamoDbAsyncClient dynamoDBClient,
@NonNull CloudWatchAsyncClient cloudWatchClient, @NonNull String workerIdentifier,
@NonNull ShardRecordProcessorFactory shardRecordProcessorFactory) {
this(new SingleStreamTracker(streamArn),
public ConfigsBuilder(
@NonNull Arn streamArn,
@NonNull String applicationName,
@NonNull KinesisAsyncClient kinesisClient,
@NonNull DynamoDbAsyncClient dynamoDBClient,
@NonNull CloudWatchAsyncClient cloudWatchClient,
@NonNull String workerIdentifier,
@NonNull ShardRecordProcessorFactory shardRecordProcessorFactory) {
this(
new SingleStreamTracker(streamArn),
applicationName,
kinesisClient,
dynamoDBClient,
@ -187,9 +199,13 @@ public class ConfigsBuilder {
* @param workerIdentifier
* @param shardRecordProcessorFactory
*/
public ConfigsBuilder(@NonNull StreamTracker streamTracker, @NonNull String applicationName,
@NonNull KinesisAsyncClient kinesisClient, @NonNull DynamoDbAsyncClient dynamoDBClient,
@NonNull CloudWatchAsyncClient cloudWatchClient, @NonNull String workerIdentifier,
public ConfigsBuilder(
@NonNull StreamTracker streamTracker,
@NonNull String applicationName,
@NonNull KinesisAsyncClient kinesisClient,
@NonNull DynamoDbAsyncClient dynamoDBClient,
@NonNull CloudWatchAsyncClient cloudWatchClient,
@NonNull String workerIdentifier,
@NonNull ShardRecordProcessorFactory shardRecordProcessorFactory) {
this.applicationName = applicationName;
this.kinesisClient = kinesisClient;
@ -209,8 +225,11 @@ public class ConfigsBuilder {
public void streamTracker(StreamTracker streamTracker) {
this.streamTracker = streamTracker;
this.appStreamTracker = DeprecationUtils.convert(streamTracker,
singleStreamTracker -> singleStreamTracker.streamConfigList().get(0).streamIdentifier().streamName());
this.appStreamTracker = DeprecationUtils.convert(streamTracker, singleStreamTracker -> singleStreamTracker
.streamConfigList()
.get(0)
.streamIdentifier()
.streamName());
}
/**

4
amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/DeprecationUtils.java
View file

@ -39,8 +39,7 @@ public final class DeprecationUtils {
*/
@Deprecated
public static <R> Either<MultiStreamTracker, R> convert(
StreamTracker streamTracker,
Function<SingleStreamTracker, R> converter) {
StreamTracker streamTracker, Function<SingleStreamTracker, R> converter) {
if (streamTracker instanceof MultiStreamTracker) {
return Either.left((MultiStreamTracker) streamTracker);
} else if (streamTracker instanceof SingleStreamTracker) {
@ -49,5 +48,4 @@ public final class DeprecationUtils {
throw new IllegalArgumentException("Unhandled StreamTracker: " + streamTracker);
}
}
}

20
amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/DiagnosticUtils.java
View file

@ -15,12 +15,12 @@
package software.amazon.kinesis.common;
import org.slf4j.Logger;
import software.amazon.kinesis.annotations.KinesisClientInternalApi;
import java.time.Duration;
import java.time.Instant;
import org.slf4j.Logger;
import software.amazon.kinesis.annotations.KinesisClientInternalApi;
import static software.amazon.kinesis.lifecycle.ShardConsumer.MAX_TIME_BETWEEN_REQUEST_RESPONSE;
@KinesisClientInternalApi
@ -32,18 +32,22 @@ public class DiagnosticUtils {
* @param enqueueTimestamp of the event submitted to the executor service
* @param log Slf4j Logger from RecordPublisher to log the events
*/
public static void takeDelayedDeliveryActionIfRequired(String resourceIdentifier, Instant enqueueTimestamp, Logger log) {
final long durationBetweenEnqueueAndAckInMillis = Duration
.between(enqueueTimestamp, Instant.now()).toMillis();
public static void takeDelayedDeliveryActionIfRequired(
String resourceIdentifier, Instant enqueueTimestamp, Logger log) {
final long durationBetweenEnqueueAndAckInMillis =
Duration.between(enqueueTimestamp, Instant.now()).toMillis();
if (durationBetweenEnqueueAndAckInMillis > MAX_TIME_BETWEEN_REQUEST_RESPONSE / 3) {
// The above condition logs the warn msg if the delivery time exceeds 11 seconds.
log.warn(
"{}: Record delivery time to shard consumer is high at {} millis. Check the ExecutorStateEvent logs"
+ " to see the state of the executor service. Also check if the RecordProcessor's processing "
+ "time is high. ",
resourceIdentifier, durationBetweenEnqueueAndAckInMillis);
resourceIdentifier,
durationBetweenEnqueueAndAckInMillis);
} else if (log.isDebugEnabled()) {
log.debug("{}: Record delivery time to shard consumer is {} millis", resourceIdentifier,
log.debug(
"{}: Record delivery time to shard consumer is {} millis",
resourceIdentifier,
durationBetweenEnqueueAndAckInMillis);
}
}

1
amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/FutureUtils.java
View file

@ -31,5 +31,4 @@ public class FutureUtils {
throw te;
}
}
}

21
amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/HashKeyRangeForLease.java
View file

@ -15,14 +15,14 @@
package software.amazon.kinesis.common;
import java.math.BigInteger;
import lombok.NonNull;
import lombok.Value;
import lombok.experimental.Accessors;
import org.apache.commons.lang3.Validate;
import software.amazon.awssdk.services.kinesis.model.HashKeyRange;
import java.math.BigInteger;
/**
* Lease POJO to hold the starting hashkey range and ending hashkey range of kinesis shards.
*/
@ -34,8 +34,11 @@ public class HashKeyRangeForLease {
private final BigInteger endingHashKey;
public HashKeyRangeForLease(BigInteger startingHashKey, BigInteger endingHashKey) {
Validate.isTrue(startingHashKey.compareTo(endingHashKey) < 0,
"StartingHashKey %s must be less than EndingHashKey %s ", startingHashKey, endingHashKey);
Validate.isTrue(
startingHashKey.compareTo(endingHashKey) < 0,
"StartingHashKey %s must be less than EndingHashKey %s ",
startingHashKey,
endingHashKey);
this.startingHashKey = startingHashKey;
this.endingHashKey = endingHashKey;
}
@ -65,11 +68,15 @@ public class HashKeyRangeForLease {
* @param endingHashKeyStr
* @return HashKeyRangeForLease
*/
public static HashKeyRangeForLease deserialize(@NonNull String startingHashKeyStr, @NonNull String endingHashKeyStr) {
public static HashKeyRangeForLease deserialize(
@NonNull String startingHashKeyStr, @NonNull String endingHashKeyStr) {
final BigInteger startingHashKey = new BigInteger(startingHashKeyStr);
final BigInteger endingHashKey = new BigInteger(endingHashKeyStr);
Validate.isTrue(startingHashKey.compareTo(endingHashKey) < 0,
"StartingHashKey %s must be less than EndingHashKey %s ", startingHashKeyStr, endingHashKeyStr);
Validate.isTrue(
startingHashKey.compareTo(endingHashKey) < 0,
"StartingHashKey %s must be less than EndingHashKey %s ",
startingHashKeyStr,
endingHashKeyStr);
return new HashKeyRangeForLease(startingHashKey, endingHashKey);
}

7
amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/InitialPositionInStreamExtended.java
View file

@ -14,16 +14,17 @@
*/
package software.amazon.kinesis.common;
import java.util.Date;
import lombok.EqualsAndHashCode;
import lombok.ToString;
import java.util.Date;
/**
* Class that houses the entities needed to specify the position in the stream from where a new application should
* start.
*/
@ToString @EqualsAndHashCode
@ToString
@EqualsAndHashCode
public class InitialPositionInStreamExtended {
private final InitialPositionInStream position;

13
amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/KinesisClientUtil.java
View file

@ -15,14 +15,14 @@
package software.amazon.kinesis.common;
import java.time.Duration;
import software.amazon.awssdk.http.Protocol;
import software.amazon.awssdk.http.nio.netty.Http2Configuration;
import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient;
import software.amazon.awssdk.services.kinesis.KinesisAsyncClient;
import software.amazon.awssdk.services.kinesis.KinesisAsyncClientBuilder;
import java.time.Duration;
/**
* Utility to setup KinesisAsyncClient to be used with KCL.
*/
@ -42,9 +42,12 @@ public class KinesisClientUtil {
}
public static KinesisAsyncClientBuilder adjustKinesisClientBuilder(KinesisAsyncClientBuilder builder) {
return builder.httpClientBuilder(NettyNioAsyncHttpClient.builder().maxConcurrency(Integer.MAX_VALUE)
.http2Configuration(Http2Configuration.builder().initialWindowSize(INITIAL_WINDOW_SIZE_BYTES)
.healthCheckPingPeriod(Duration.ofMillis(HEALTH_CHECK_PING_PERIOD_MILLIS)).build())
return builder.httpClientBuilder(NettyNioAsyncHttpClient.builder()
.maxConcurrency(Integer.MAX_VALUE)
.http2Configuration(Http2Configuration.builder()
.initialWindowSize(INITIAL_WINDOW_SIZE_BYTES)
.healthCheckPingPeriod(Duration.ofMillis(HEALTH_CHECK_PING_PERIOD_MILLIS))
.build())
.protocol(Protocol.HTTP2));
}
}

11
amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/KinesisRequestsBuilder.java
View file

@ -63,12 +63,11 @@ public class KinesisRequestsBuilder {
@SuppressWarnings("unchecked")
private static <T extends AwsRequest.Builder> T appendUserAgent(final T builder) {
return (T) builder
.overrideConfiguration(
AwsRequestOverrideConfiguration.builder()
.addApiName(ApiName.builder().name(RetrievalConfig.KINESIS_CLIENT_LIB_USER_AGENT)
.version(RetrievalConfig.KINESIS_CLIENT_LIB_USER_AGENT_VERSION).build())
return (T) builder.overrideConfiguration(AwsRequestOverrideConfiguration.builder()
.addApiName(ApiName.builder()
.name(RetrievalConfig.KINESIS_CLIENT_LIB_USER_AGENT)
.version(RetrievalConfig.KINESIS_CLIENT_LIB_USER_AGENT_VERSION)
.build())
.build());
}
}

2
amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/LeaseCleanupConfig.java
View file

@ -24,7 +24,7 @@ import lombok.experimental.Accessors;
*/
@Builder
@Getter
@Accessors(fluent=true)
@Accessors(fluent = true)
public class LeaseCleanupConfig {
/**
* Interval at which to run lease cleanup thread.

8
amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/RequestDetails.java
View file

@ -15,11 +15,11 @@
package software.amazon.kinesis.common;
import lombok.experimental.Accessors;
import java.util.Optional;
@Accessors(fluent=true)
import lombok.experimental.Accessors;
@Accessors(fluent = true)
public class RequestDetails {
/**
@ -62,6 +62,4 @@ public class RequestDetails {
public String toString() {
return String.format("request id - %s, timestamp - %s", getRequestId(), getTimestamp());
}
}

3
amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/StreamConfig.java
View file

@ -28,8 +28,7 @@ import lombok.experimental.Accessors;
public class StreamConfig {
@NonNull
private final StreamIdentifier streamIdentifier;
private final InitialPositionInStreamExtended initialPositionInStreamExtended;
private String consumerArn;
}

32
amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/StreamIdentifier.java
View file

@ -15,6 +15,10 @@
package software.amazon.kinesis.common;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import lombok.AccessLevel;
import lombok.Builder;
import lombok.EqualsAndHashCode;
@ -25,10 +29,6 @@ import software.amazon.awssdk.arns.Arn;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.utils.Validate;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@Builder(access = AccessLevel.PRIVATE)
@EqualsAndHashCode
@Getter
@ -37,10 +37,13 @@ public class StreamIdentifier {
@Builder.Default
private final Optional<String> accountIdOptional = Optional.empty();
@NonNull
private final String streamName;
@Builder.Default
private final Optional<Long> streamCreationEpochOptional = Optional.empty();
@Builder.Default
@EqualsAndHashCode.Exclude
private final Optional<Arn> streamArnOptional = Optional.empty();
@ -49,8 +52,8 @@ public class StreamIdentifier {
* Pattern for a serialized {@link StreamIdentifier}. The valid format is
* {@code <accountId>:<streamName>:<creationEpoch>}.
*/
private static final Pattern STREAM_IDENTIFIER_PATTERN = Pattern.compile(
"(?<accountId>[0-9]+):(?<streamName>[^:]+):(?<creationEpoch>[0-9]+)");
private static final Pattern STREAM_IDENTIFIER_PATTERN =
Pattern.compile("(?<accountId>[0-9]+):(?<streamName>[^:]+):(?<creationEpoch>[0-9]+)");
/**
* Pattern for a stream ARN. The valid format is
@ -74,8 +77,10 @@ public class StreamIdentifier {
final char delimiter = ':';
final StringBuilder sb = new StringBuilder()
.append(accountIdOptional.get()).append(delimiter)
.append(streamName).append(delimiter)
.append(accountIdOptional.get())
.append(delimiter)
.append(streamName)
.append(delimiter)
.append(streamCreationEpochOptional.get());
return sb.toString();
}
@ -146,9 +151,7 @@ public class StreamIdentifier {
public static StreamIdentifier singleStreamInstance(String streamName) {
Validate.notEmpty(streamName, "StreamName should not be empty");
return StreamIdentifier.builder()
.streamName(streamName)
.build();
return StreamIdentifier.builder().streamName(streamName).build();
}
/**
@ -173,7 +176,8 @@ public class StreamIdentifier {
* @param streamArn
*/
public static void validateArn(Arn streamArn) {
if (!STREAM_ARN_PATTERN.matcher(streamArn.toString()).matches() || !streamArn.region().isPresent()) {
if (!STREAM_ARN_PATTERN.matcher(streamArn.toString()).matches()
|| !streamArn.region().isPresent()) {
throw new IllegalArgumentException("Invalid streamArn " + streamArn);
}
}
@ -185,9 +189,7 @@ public class StreamIdentifier {
*/
private static void validateCreationEpoch(long creationEpoch) {
if (creationEpoch <= 0) {
throw new IllegalArgumentException(
"Creation epoch must be > 0; received " + creationEpoch);
throw new IllegalArgumentException("Creation epoch must be > 0; received " + creationEpoch);
}
}
}

1
amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/CoordinatorConfig.java
View file

@ -96,5 +96,4 @@ public class CoordinatorConfig {
* <p>Default value: 1000L</p>
*/
private long schedulerInitializationBackoffTimeMillis = 1000L;
}

1
amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/DeletedStreamListProvider.java
View file

@ -5,7 +5,6 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import lombok.extern.slf4j.Slf4j;
import software.amazon.kinesis.common.StreamIdentifier;
/**

43
amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/DeterministicShuffleShardSyncLeaderDecider.java
View file

@ -26,6 +26,7 @@ import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.BooleanSupplier;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import software.amazon.awssdk.utils.CollectionUtils;
import software.amazon.kinesis.leases.Lease;
@ -45,8 +46,7 @@ import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException;
* This ensures redundancy for shard-sync during host failures.
*/
@Slf4j
class DeterministicShuffleShardSyncLeaderDecider
implements LeaderDecider {
class DeterministicShuffleShardSyncLeaderDecider implements LeaderDecider {
// Fixed seed so that the shuffle order is preserved across workers
static final int DETERMINISTIC_SHUFFLE_SEED = 1947;
@ -67,13 +67,11 @@ class DeterministicShuffleShardSyncLeaderDecider
* @param leaderElectionThreadPool Thread-pool to be used for leaderElection.
* @param numPeriodicShardSyncWorkers Number of leaders that will be elected to perform periodic shard syncs.
*/
DeterministicShuffleShardSyncLeaderDecider(LeaseRefresher leaseRefresher,
ScheduledExecutorService leaderElectionThreadPool,
int numPeriodicShardSyncWorkers) {
this(leaseRefresher,
leaderElectionThreadPool,
numPeriodicShardSyncWorkers,
new ReentrantReadWriteLock());
DeterministicShuffleShardSyncLeaderDecider(
LeaseRefresher leaseRefresher,
ScheduledExecutorService leaderElectionThreadPool,
int numPeriodicShardSyncWorkers) {
this(leaseRefresher, leaderElectionThreadPool, numPeriodicShardSyncWorkers, new ReentrantReadWriteLock());
}
/**
@ -82,10 +80,11 @@ class DeterministicShuffleShardSyncLeaderDecider
* @param numPeriodicShardSyncWorkers Number of leaders that will be elected to perform periodic shard syncs.
* @param readWriteLock Mechanism to lock for reading and writing of critical components
*/
DeterministicShuffleShardSyncLeaderDecider(LeaseRefresher leaseRefresher,
ScheduledExecutorService leaderElectionThreadPool,
int numPeriodicShardSyncWorkers,
ReadWriteLock readWriteLock) {
DeterministicShuffleShardSyncLeaderDecider(
LeaseRefresher leaseRefresher,
ScheduledExecutorService leaderElectionThreadPool,
int numPeriodicShardSyncWorkers,
ReadWriteLock readWriteLock) {
this.leaseRefresher = leaseRefresher;
this.leaderElectionThreadPool = leaderElectionThreadPool;
this.numPeriodicShardSyncWorkers = numPeriodicShardSyncWorkers;
@ -101,8 +100,12 @@ class DeterministicShuffleShardSyncLeaderDecider
try {
log.debug("Started leader election at: " + Instant.now());
List<Lease> leases = leaseRefresher.listLeases();
List<String> uniqueHosts = leases.stream().map(Lease::leaseOwner)
.filter(owner -> owner != null).distinct().sorted().collect(Collectors.toList());
List<String> uniqueHosts = leases.stream()
.map(Lease::leaseOwner)
.filter(owner -> owner != null)
.distinct()
.sorted()
.collect(Collectors.toList());
Collections.shuffle(uniqueHosts, new Random(DETERMINISTIC_SHUFFLE_SEED));
int numShardSyncWorkers = Math.min(uniqueHosts.size(), numPeriodicShardSyncWorkers);
@ -137,8 +140,11 @@ class DeterministicShuffleShardSyncLeaderDecider
// The first run will be after a minute.
// We don't need jitter since it is scheduled with a fixed delay and time taken to scan leases
// will be different at different times and on different hosts/workers.
leaderElectionThreadPool.scheduleWithFixedDelay(this::electLeaders, ELECTION_INITIAL_DELAY_MILLIS,
ELECTION_SCHEDULING_INTERVAL_MILLIS, TimeUnit.MILLISECONDS);
leaderElectionThreadPool.scheduleWithFixedDelay(
this::electLeaders,
ELECTION_INITIAL_DELAY_MILLIS,
ELECTION_SCHEDULING_INTERVAL_MILLIS,
TimeUnit.MILLISECONDS);
}
return executeConditionCheckWithReadLock(() -> isWorkerLeaderForShardSync(workerId));
@ -152,7 +158,8 @@ class DeterministicShuffleShardSyncLeaderDecider
log.info("Successfully stopped leader election on the worker");
} else {
leaderElectionThreadPool.shutdownNow();
log.info(String.format("Stopped leader election thread after awaiting termination for %d milliseconds",
log.info(String.format(
"Stopped leader election thread after awaiting termination for %d milliseconds",
AWAIT_TERMINATION_MILLIS));
}

4
amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/DiagnosticEventFactory.java
View file

@ -15,11 +15,11 @@
package software.amazon.kinesis.coordinator;
import java.util.concurrent.ExecutorService;
import lombok.NoArgsConstructor;
import software.amazon.kinesis.leases.LeaseCoordinator;
import java.util.concurrent.ExecutorService;
/**
* Creates {@link DiagnosticEvent}s for logging and visibility
*/

6
amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/ExecutorStateEvent.java
View file

@ -15,15 +15,15 @@
package software.amazon.kinesis.coordinator;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadPoolExecutor;
import lombok.Getter;
import lombok.ToString;
import lombok.extern.slf4j.Slf4j;
import software.amazon.kinesis.annotations.KinesisClientInternalApi;
import software.amazon.kinesis.leases.LeaseCoordinator;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadPoolExecutor;
@Getter
@ToString(exclude = "isThreadPoolExecutor")
@Slf4j

4
amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/GracefulShutdownContext.java
View file

@ -14,12 +14,12 @@
*/
package software.amazon.kinesis.coordinator;
import java.util.concurrent.CountDownLatch;
import lombok.Builder;
import lombok.Data;
import lombok.experimental.Accessors;
import java.util.concurrent.CountDownLatch;
@Data
@Builder
@Accessors(fluent = true)

36
amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/GracefulShutdownCoordinator.java
View file

@ -14,13 +14,12 @@
*/
package software.amazon.kinesis.coordinator;
import lombok.extern.slf4j.Slf4j;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
class GracefulShutdownCoordinator {
/**
@ -31,8 +30,11 @@ class GracefulShutdownCoordinator {
CompletableFuture<Boolean> startGracefulShutdown(Callable<Boolean> shutdownCallable) {
CompletableFuture<Boolean> cf = new CompletableFuture<>();
CompletableFuture.runAsync(() -> {
try { cf.complete(shutdownCallable.call()); }
catch(Throwable ex) { cf.completeExceptionally(ex); }
try {
cf.complete(shutdownCallable.call());
} catch (Throwable ex) {
cf.completeExceptionally(ex);
}
});
return cf;
}
@ -50,7 +52,8 @@ class GracefulShutdownCoordinator {
}
private boolean isWorkerShutdownComplete(GracefulShutdownContext context) {
return context.scheduler().shutdownComplete() || context.scheduler().shardInfoShardConsumerMap().isEmpty();
return context.scheduler().shutdownComplete()
|| context.scheduler().shardInfoShardConsumerMap().isEmpty();
}
private String awaitingLogMessage(GracefulShutdownContext context) {
@ -92,12 +95,14 @@ class GracefulShutdownCoordinator {
throw new InterruptedException();
}
log.info(awaitingLogMessage(context));
if (workerShutdownWithRemaining(context.shutdownCompleteLatch().getCount(), context)) {
if (workerShutdownWithRemaining(
context.shutdownCompleteLatch().getCount(), context)) {
return false;
}
}
} catch (InterruptedException ie) {
log.warn("Interrupted while waiting for notification complete, terminating shutdown. {}",
log.warn(
"Interrupted while waiting for notification complete, terminating shutdown. {}",
awaitingLogMessage(context));
return false;
}
@ -129,12 +134,14 @@ class GracefulShutdownCoordinator {
throw new InterruptedException();
}
log.info(awaitingFinalShutdownMessage(context));
if (workerShutdownWithRemaining(context.shutdownCompleteLatch().getCount(), context)) {
if (workerShutdownWithRemaining(
context.shutdownCompleteLatch().getCount(), context)) {
return false;
}
}
} catch (InterruptedException ie) {
log.warn("Interrupted while waiting for shutdown completion, terminating shutdown. {}",
log.warn(
"Interrupted while waiting for shutdown completion, terminating shutdown. {}",
awaitingFinalShutdownMessage(context));
return false;
}
@ -152,9 +159,12 @@ class GracefulShutdownCoordinator {
private boolean workerShutdownWithRemaining(long outstanding, GracefulShutdownContext context) {
if (isWorkerShutdownComplete(context)) {
if (outstanding != 0) {
log.info("Shutdown completed, but shutdownCompleteLatch still had outstanding {} with a current"
+ " value of {}. shutdownComplete: {} -- Consumer Map: {}", outstanding,
context.shutdownCompleteLatch().getCount(), context.scheduler().shutdownComplete(),
log.info(
"Shutdown completed, but shutdownCompleteLatch still had outstanding {} with a current"
+ " value of {}. shutdownComplete: {} -- Consumer Map: {}",
outstanding,
context.shutdownCompleteLatch().getCount(),
context.scheduler().shutdownComplete(),
context.scheduler().shardInfoShardConsumerMap().size());
return true;
}

8
amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/NoOpWorkerStateChangeListener.java
View file

@ -19,12 +19,8 @@ public class NoOpWorkerStateChangeListener implements WorkerStateChangeListener
/**
* Empty constructor for NoOp Worker State Change Listener
*/
public NoOpWorkerStateChangeListener() {
}
public NoOpWorkerStateChangeListener() {}
@Override
public void onWorkerStateChange(WorkerState newState) {
}
public void onWorkerStateChange(WorkerState newState) {}
}

268
amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/PeriodicShardSyncManager.java
View file

@ -14,6 +14,24 @@
*/
package software.amazon.kinesis.coordinator;
import java.io.Serializable;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Function;
import java.util.stream.Collectors;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ComparisonChain;
import lombok.AccessLevel;
@ -46,24 +64,6 @@ import software.amazon.kinesis.metrics.MetricsLevel;
import software.amazon.kinesis.metrics.MetricsScope;
import software.amazon.kinesis.metrics.MetricsUtil;
import java.io.Serializable;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Function;
import java.util.stream.Collectors;
import static software.amazon.kinesis.common.HashKeyRangeForLease.fromHashKeyRange;
/**
@ -76,10 +76,13 @@ import static software.amazon.kinesis.common.HashKeyRangeForLease.fromHashKeyRan
@KinesisClientInternalApi
class PeriodicShardSyncManager {
private static final long INITIAL_DELAY = 60 * 1000L;
@VisibleForTesting
static final BigInteger MIN_HASH_KEY = BigInteger.ZERO;
@VisibleForTesting
static final BigInteger MAX_HASH_KEY = new BigInteger("2").pow(128).subtract(BigInteger.ONE);
static final String PERIODIC_SHARD_SYNC_MANAGER = "PeriodicShardSyncManager";
private final Map<StreamIdentifier, HashRangeHoleTracker> hashRangeHoleTrackerMap = new HashMap<>();
@ -94,30 +97,48 @@ class PeriodicShardSyncManager {
private final MetricsFactory metricsFactory;
private final long leasesRecoveryAuditorExecutionFrequencyMillis;
private final int leasesRecoveryAuditorInconsistencyConfidenceThreshold;
@Getter(AccessLevel.NONE)
private final AtomicBoolean leaderSynced;
private boolean isRunning;
PeriodicShardSyncManager(String workerId, LeaderDecider leaderDecider, LeaseRefresher leaseRefresher,
PeriodicShardSyncManager(
String workerId,
LeaderDecider leaderDecider,
LeaseRefresher leaseRefresher,
Map<StreamIdentifier, StreamConfig> currentStreamConfigMap,
Function<StreamConfig, ShardSyncTaskManager> shardSyncTaskManagerProvider,
Map<StreamConfig, ShardSyncTaskManager> streamToShardSyncTaskManagerMap,
boolean isMultiStreamingMode, MetricsFactory metricsFactory,
boolean isMultiStreamingMode,
MetricsFactory metricsFactory,
long leasesRecoveryAuditorExecutionFrequencyMillis,
int leasesRecoveryAuditorInconsistencyConfidenceThreshold,
AtomicBoolean leaderSynced){
this(workerId, leaderDecider, leaseRefresher, currentStreamConfigMap, shardSyncTaskManagerProvider,
AtomicBoolean leaderSynced) {
this(
workerId,
leaderDecider,
leaseRefresher,
currentStreamConfigMap,
shardSyncTaskManagerProvider,
streamToShardSyncTaskManagerMap,
Executors.newSingleThreadScheduledExecutor(), isMultiStreamingMode, metricsFactory,
leasesRecoveryAuditorExecutionFrequencyMillis, leasesRecoveryAuditorInconsistencyConfidenceThreshold,
Executors.newSingleThreadScheduledExecutor(),
isMultiStreamingMode,
metricsFactory,
leasesRecoveryAuditorExecutionFrequencyMillis,
leasesRecoveryAuditorInconsistencyConfidenceThreshold,
leaderSynced);
}
PeriodicShardSyncManager(String workerId, LeaderDecider leaderDecider, LeaseRefresher leaseRefresher,
PeriodicShardSyncManager(
String workerId,
LeaderDecider leaderDecider,
LeaseRefresher leaseRefresher,
Map<StreamIdentifier, StreamConfig> currentStreamConfigMap,
Function<StreamConfig, ShardSyncTaskManager> shardSyncTaskManagerProvider,
Map<StreamConfig, ShardSyncTaskManager> streamToShardSyncTaskManagerMap,
ScheduledExecutorService shardSyncThreadPool, boolean isMultiStreamingMode,
ScheduledExecutorService shardSyncThreadPool,
boolean isMultiStreamingMode,
MetricsFactory metricsFactory,
long leasesRecoveryAuditorExecutionFrequencyMillis,
int leasesRecoveryAuditorInconsistencyConfidenceThreshold,
@ -134,7 +155,8 @@ class PeriodicShardSyncManager {
this.isMultiStreamingMode = isMultiStreamingMode;
this.metricsFactory = metricsFactory;
this.leasesRecoveryAuditorExecutionFrequencyMillis = leasesRecoveryAuditorExecutionFrequencyMillis;
this.leasesRecoveryAuditorInconsistencyConfidenceThreshold = leasesRecoveryAuditorInconsistencyConfidenceThreshold;
this.leasesRecoveryAuditorInconsistencyConfidenceThreshold =
leasesRecoveryAuditorInconsistencyConfidenceThreshold;
this.leaderSynced = leaderSynced;
}
@ -147,10 +169,12 @@ class PeriodicShardSyncManager {
log.error("Error during runShardSync.", t);
}
};
shardSyncThreadPool.scheduleWithFixedDelay(periodicShardSyncer, INITIAL_DELAY, leasesRecoveryAuditorExecutionFrequencyMillis,
shardSyncThreadPool.scheduleWithFixedDelay(
periodicShardSyncer,
INITIAL_DELAY,
leasesRecoveryAuditorExecutionFrequencyMillis,
TimeUnit.MILLISECONDS);
isRunning = true;
}
return new TaskResult(null);
}
@ -186,8 +210,8 @@ class PeriodicShardSyncManager {
if (leaderDecider.isLeader(workerId) && leaderSynced.get()) {
log.info(String.format("WorkerId %s is leader, running the periodic shard sync task", workerId));
final MetricsScope scope = MetricsUtil.createMetricsWithOperation(metricsFactory,
PERIODIC_SHARD_SYNC_MANAGER);
final MetricsScope scope =
MetricsUtil.createMetricsWithOperation(metricsFactory, PERIODIC_SHARD_SYNC_MANAGER);
int numStreamsWithPartialLeases = 0;
int numStreamsToSync = 0;
int numSkippedShardSyncTask = 0;
@ -207,15 +231,17 @@ class PeriodicShardSyncManager {
log.info("Skipping shard sync task for {} as stream is purged", streamIdentifier);
continue;
}
final ShardSyncResponse shardSyncResponse = checkForShardSync(streamIdentifier,
streamToLeasesMap.get(streamIdentifier));
final ShardSyncResponse shardSyncResponse =
checkForShardSync(streamIdentifier, streamToLeasesMap.get(streamIdentifier));
numStreamsWithPartialLeases += shardSyncResponse.isHoleDetected() ? 1 : 0;
numStreamsToSync += shardSyncResponse.shouldDoShardSync ? 1 : 0;
if (shardSyncResponse.shouldDoShardSync()) {
log.info("Periodic shard syncer initiating shard sync for {} due to the reason - {} ",
streamIdentifier, shardSyncResponse.reasonForDecision());
log.info(
"Periodic shard syncer initiating shard sync for {} due to the reason - {} ",
streamIdentifier,
shardSyncResponse.reasonForDecision());
final StreamConfig streamConfig = currentStreamConfigMap.get(streamIdentifier);
if (streamConfig == null) {
log.info("Skipping shard sync task for {} as stream is purged", streamIdentifier);
@ -223,15 +249,15 @@ class PeriodicShardSyncManager {
}
final ShardSyncTaskManager shardSyncTaskManager;
if (streamToShardSyncTaskManagerMap.containsKey(streamConfig)) {
log.info("shardSyncTaskManager for stream {} already exists",
streamIdentifier.streamName());
log.info(
"shardSyncTaskManager for stream {} already exists", streamIdentifier.streamName());
shardSyncTaskManager = streamToShardSyncTaskManagerMap.get(streamConfig);
}
else {
} else {
// If streamConfig of a stream has already been added to currentStreamConfigMap but
// Scheduler failed to create shardSyncTaskManager for it, then Scheduler will not try
// to create one later. So enable PeriodicShardSyncManager to do it for such cases
log.info("Failed to get shardSyncTaskManager so creating one for stream {}.",
log.info(
"Failed to get shardSyncTaskManager so creating one for stream {}.",
streamIdentifier.streamName());
shardSyncTaskManager = streamToShardSyncTaskManagerMap.computeIfAbsent(
streamConfig, s -> shardSyncTaskManagerProvider.apply(s));
@ -239,15 +265,24 @@ class PeriodicShardSyncManager {
if (!shardSyncTaskManager.submitShardSyncTask()) {
log.warn(
"Failed to submit shard sync task for stream {}. This could be due to the previous pending shard sync task.",
shardSyncTaskManager.shardDetector().streamIdentifier().streamName());
shardSyncTaskManager
.shardDetector()
.streamIdentifier()
.streamName());
numSkippedShardSyncTask += 1;
} else {
log.info("Submitted shard sync task for stream {} because of reason {}",
shardSyncTaskManager.shardDetector().streamIdentifier().streamName(),
log.info(
"Submitted shard sync task for stream {} because of reason {}",
shardSyncTaskManager
.shardDetector()
.streamIdentifier()
.streamName(),
shardSyncResponse.reasonForDecision());
}
} else {
log.info("Skipping shard sync for {} due to the reason - {}", streamIdentifier,
log.info(
"Skipping shard sync for {} due to the reason - {}",
streamIdentifier,
shardSyncResponse.reasonForDecision());
}
}
@ -255,9 +290,14 @@ class PeriodicShardSyncManager {
} catch (Exception e) {
log.error("Caught exception while running periodic shard syncer.", e);
} finally {
scope.addData("NumStreamsWithPartialLeases", numStreamsWithPartialLeases, StandardUnit.COUNT, MetricsLevel.SUMMARY);
scope.addData(
"NumStreamsWithPartialLeases",
numStreamsWithPartialLeases,
StandardUnit.COUNT,
MetricsLevel.SUMMARY);
scope.addData("NumStreamsToSync", numStreamsToSync, StandardUnit.COUNT, MetricsLevel.SUMMARY);
scope.addData("NumSkippedShardSyncTask", numSkippedShardSyncTask, StandardUnit.COUNT, MetricsLevel.SUMMARY);
scope.addData(
"NumSkippedShardSyncTask", numSkippedShardSyncTask, StandardUnit.COUNT, MetricsLevel.SUMMARY);
MetricsUtil.addSuccessAndLatency(scope, isRunSuccess, runStartMillis, MetricsLevel.SUMMARY);
scope.end();
}
@ -284,17 +324,18 @@ class PeriodicShardSyncManager {
} else {
final Map<StreamIdentifier, List<Lease>> streamToLeasesMap = new HashMap<>();
for (Lease lease : leases) {
StreamIdentifier streamIdentifier = StreamIdentifier
.multiStreamInstance(((MultiStreamLease) lease).streamIdentifier());
StreamIdentifier streamIdentifier =
StreamIdentifier.multiStreamInstance(((MultiStreamLease) lease).streamIdentifier());
if (streamIdentifiersToFilter.contains(streamIdentifier)) {
streamToLeasesMap.computeIfAbsent(streamIdentifier, s -> new ArrayList<>()).add(lease);
streamToLeasesMap
.computeIfAbsent(streamIdentifier, s -> new ArrayList<>())
.add(lease);
}
}
return streamToLeasesMap;
}
}
/**
* Given a list of leases for a stream, determine if a shard sync is necessary.
* @param streamIdentifier
@ -315,11 +356,13 @@ class PeriodicShardSyncManager {
// If hole is determined with high confidence return true; return false otherwise
// We are using the high confidence factor to avoid shard sync on any holes during resharding and
// lease cleanups or any intermittent issues.
final HashRangeHoleTracker hashRangeHoleTracker = hashRangeHoleTrackerMap
.computeIfAbsent(streamIdentifier, s -> new HashRangeHoleTracker());
final boolean hasHoleWithHighConfidence = hashRangeHoleTracker
.hasHighConfidenceOfHoleWith(hashRangeHoleOpt.get());
return new ShardSyncResponse(hasHoleWithHighConfidence, true,
final HashRangeHoleTracker hashRangeHoleTracker =
hashRangeHoleTrackerMap.computeIfAbsent(streamIdentifier, s -> new HashRangeHoleTracker());
final boolean hasHoleWithHighConfidence =
hashRangeHoleTracker.hasHighConfidenceOfHoleWith(hashRangeHoleOpt.get());
return new ShardSyncResponse(
hasHoleWithHighConfidence,
true,
"Detected same hole for " + hashRangeHoleTracker.getNumConsecutiveHoles()
+ " times. Shard sync will be initiated when threshold reaches "
+ leasesRecoveryAuditorInconsistencyConfidenceThreshold);
@ -355,7 +398,9 @@ class PeriodicShardSyncManager {
Optional<HashRangeHole> hasHoleInLeases(StreamIdentifier streamIdentifier, List<Lease> leases) {
// Filter the leases with any checkpoint other than shard end.
List<Lease> activeLeases = leases.stream()
.filter(lease -> lease.checkpoint() != null && !lease.checkpoint().isShardEnd()).collect(Collectors.toList());
.filter(lease ->
lease.checkpoint() != null && !lease.checkpoint().isShardEnd())
.collect(Collectors.toList());
List<Lease> activeLeasesWithHashRanges = fillWithHashRangesIfRequired(streamIdentifier, activeLeases);
return checkForHoleInHashKeyRanges(streamIdentifier, activeLeasesWithHashRanges);
}
@ -364,43 +409,49 @@ class PeriodicShardSyncManager {
// by learning from kinesis shards.
private List<Lease> fillWithHashRangesIfRequired(StreamIdentifier streamIdentifier, List<Lease> activeLeases) {
List<Lease> activeLeasesWithNoHashRanges = activeLeases.stream()
.filter(lease -> lease.hashKeyRangeForLease() == null).collect(Collectors.toList());
.filter(lease -> lease.hashKeyRangeForLease() == null)
.collect(Collectors.toList());
Optional<Lease> minLeaseOpt = activeLeasesWithNoHashRanges.stream().min(Comparator.comparing(Lease::leaseKey));
if (minLeaseOpt.isPresent()) {
// TODO : use minLease for new ListShards with startingShardId
final Lease minLease = minLeaseOpt.get();
final ShardDetector shardDetector = shardSyncTaskManagerProvider
.apply(currentStreamConfigMap.get(streamIdentifier)).shardDetector();
final Map<String, Shard> kinesisShards = shardDetector.listShards().stream()
.collect(Collectors.toMap(Shard::shardId, shard -> shard));
return activeLeases.stream().map(lease -> {
if (lease.hashKeyRangeForLease() == null) {
final String shardId = lease instanceof MultiStreamLease ?
((MultiStreamLease) lease).shardId() :
lease.leaseKey();
final Shard shard = kinesisShards.get(shardId);
if (shard == null) {
.apply(currentStreamConfigMap.get(streamIdentifier))
.shardDetector();
final Map<String, Shard> kinesisShards =
shardDetector.listShards().stream().collect(Collectors.toMap(Shard::shardId, shard -> shard));
return activeLeases.stream()
.map(lease -> {
if (lease.hashKeyRangeForLease() == null) {
final String shardId = lease instanceof MultiStreamLease
? ((MultiStreamLease) lease).shardId()
: lease.leaseKey();
final Shard shard = kinesisShards.get(shardId);
if (shard == null) {
return lease;
}
lease.hashKeyRange(fromHashKeyRange(shard.hashKeyRange()));
try {
leaseRefresher.updateLeaseWithMetaInfo(lease, UpdateField.HASH_KEY_RANGE);
} catch (Exception e) {
log.warn(
"Unable to update hash range key information for lease {} of stream {}. This may result in explicit lease sync.",
lease.leaseKey(),
streamIdentifier);
}
}
return lease;
}
lease.hashKeyRange(fromHashKeyRange(shard.hashKeyRange()));
try {
leaseRefresher.updateLeaseWithMetaInfo(lease, UpdateField.HASH_KEY_RANGE);
} catch (Exception e) {
log.warn(
"Unable to update hash range key information for lease {} of stream {}. This may result in explicit lease sync.",
lease.leaseKey(), streamIdentifier);
}
}
return lease;
}).filter(lease -> lease.hashKeyRangeForLease() != null).collect(Collectors.toList());
})
.filter(lease -> lease.hashKeyRangeForLease() != null)
.collect(Collectors.toList());
} else {
return activeLeases;
}
}
@VisibleForTesting
static Optional<HashRangeHole> checkForHoleInHashKeyRanges(StreamIdentifier streamIdentifier,
List<Lease> leasesWithHashKeyRanges) {
static Optional<HashRangeHole> checkForHoleInHashKeyRanges(
StreamIdentifier streamIdentifier, List<Lease> leasesWithHashKeyRanges) {
// Sort the hash ranges by starting hash key.
List<Lease> sortedLeasesWithHashKeyRanges = sortLeasesByHashRange(leasesWithHashKeyRanges);
if (sortedLeasesWithHashKeyRanges.isEmpty()) {
@ -408,34 +459,54 @@ class PeriodicShardSyncManager {
return Optional.of(new HashRangeHole());
}
// Validate for hashranges bounds.
if (!sortedLeasesWithHashKeyRanges.get(0).hashKeyRangeForLease().startingHashKey().equals(MIN_HASH_KEY) || !sortedLeasesWithHashKeyRanges
.get(sortedLeasesWithHashKeyRanges.size() - 1).hashKeyRangeForLease().endingHashKey().equals(MAX_HASH_KEY)) {
log.error("Incomplete hash range found for stream {} between {} and {}.", streamIdentifier,
if (!sortedLeasesWithHashKeyRanges
.get(0)
.hashKeyRangeForLease()
.startingHashKey()
.equals(MIN_HASH_KEY)
|| !sortedLeasesWithHashKeyRanges
.get(sortedLeasesWithHashKeyRanges.size() - 1)
.hashKeyRangeForLease()
.endingHashKey()
.equals(MAX_HASH_KEY)) {
log.error(
"Incomplete hash range found for stream {} between {} and {}.",
streamIdentifier,
sortedLeasesWithHashKeyRanges.get(0),
sortedLeasesWithHashKeyRanges.get(sortedLeasesWithHashKeyRanges.size() - 1));
return Optional.of(new HashRangeHole(sortedLeasesWithHashKeyRanges.get(0).hashKeyRangeForLease(),
sortedLeasesWithHashKeyRanges.get(sortedLeasesWithHashKeyRanges.size() - 1).hashKeyRangeForLease()));
return Optional.of(new HashRangeHole(
sortedLeasesWithHashKeyRanges.get(0).hashKeyRangeForLease(),
sortedLeasesWithHashKeyRanges
.get(sortedLeasesWithHashKeyRanges.size() - 1)
.hashKeyRangeForLease()));
}
// Check for any holes in the sorted hashrange intervals.
if (sortedLeasesWithHashKeyRanges.size() > 1) {
Lease leftMostLeaseToReportInCaseOfHole = sortedLeasesWithHashKeyRanges.get(0);
HashKeyRangeForLease leftLeaseHashRange = leftMostLeaseToReportInCaseOfHole.hashKeyRangeForLease();
for (int i = 1; i < sortedLeasesWithHashKeyRanges.size(); i++) {
final HashKeyRangeForLease rightLeaseHashRange = sortedLeasesWithHashKeyRanges.get(i).hashKeyRangeForLease();
final BigInteger rangeDiff = rightLeaseHashRange.startingHashKey().subtract(leftLeaseHashRange.endingHashKey());
final HashKeyRangeForLease rightLeaseHashRange =
sortedLeasesWithHashKeyRanges.get(i).hashKeyRangeForLease();
final BigInteger rangeDiff =
rightLeaseHashRange.startingHashKey().subtract(leftLeaseHashRange.endingHashKey());
// Case of overlapping leases when the rangediff is 0 or negative.
// signum() will be -1 for negative and 0 if value is 0.
// Merge the range for further tracking.
if (rangeDiff.signum() <= 0) {
leftLeaseHashRange = new HashKeyRangeForLease(leftLeaseHashRange.startingHashKey(),
leftLeaseHashRange = new HashKeyRangeForLease(
leftLeaseHashRange.startingHashKey(),
leftLeaseHashRange.endingHashKey().max(rightLeaseHashRange.endingHashKey()));
} else {
// Case of non overlapping leases when rangediff is positive. signum() will be 1 for positive.
// If rangeDiff is 1, then it is a case of continuous hashrange. If not, it is a hole.
if (!rangeDiff.equals(BigInteger.ONE)) {
log.error("Incomplete hash range found for {} between {} and {}.", streamIdentifier,
leftMostLeaseToReportInCaseOfHole, sortedLeasesWithHashKeyRanges.get(i));
return Optional.of(new HashRangeHole(leftMostLeaseToReportInCaseOfHole.hashKeyRangeForLease(),
log.error(
"Incomplete hash range found for {} between {} and {}.",
streamIdentifier,
leftMostLeaseToReportInCaseOfHole,
sortedLeasesWithHashKeyRanges.get(i));
return Optional.of(new HashRangeHole(
leftMostLeaseToReportInCaseOfHole.hashKeyRangeForLease(),
sortedLeasesWithHashKeyRanges.get(i).hashKeyRangeForLease()));
}
leftMostLeaseToReportInCaseOfHole = sortedLeasesWithHashKeyRanges.get(i);
@ -461,7 +532,9 @@ class PeriodicShardSyncManager {
hashRangeAtStartOfPossibleHole = hashRangeAtEndOfPossibleHole = null;
}
HashRangeHole(HashKeyRangeForLease hashRangeAtStartOfPossibleHole, HashKeyRangeForLease hashRangeAtEndOfPossibleHole) {
HashRangeHole(
HashKeyRangeForLease hashRangeAtStartOfPossibleHole,
HashKeyRangeForLease hashRangeAtEndOfPossibleHole) {
this.hashRangeAtStartOfPossibleHole = hashRangeAtStartOfPossibleHole;
this.hashRangeAtEndOfPossibleHole = hashRangeAtEndOfPossibleHole;
}
@ -472,6 +545,7 @@ class PeriodicShardSyncManager {
private class HashRangeHoleTracker {
private HashRangeHole hashRangeHole;
@Getter
private Integer numConsecutiveHoles;
@ -500,8 +574,12 @@ class PeriodicShardSyncManager {
Validate.notNull(lease.hashKeyRangeForLease());
Validate.notNull(otherLease.hashKeyRangeForLease());
return ComparisonChain.start()
.compare(lease.hashKeyRangeForLease().startingHashKey(), otherLease.hashKeyRangeForLease().startingHashKey())
.compare(lease.hashKeyRangeForLease().endingHashKey(), otherLease.hashKeyRangeForLease().endingHashKey())
.compare(
lease.hashKeyRangeForLease().startingHashKey(),
otherLease.hashKeyRangeForLease().startingHashKey())
.compare(
lease.hashKeyRangeForLease().endingHashKey(),
otherLease.hashKeyRangeForLease().endingHashKey())
.result();
}
}

4
amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/RejectedTaskEvent.java
View file

@ -25,8 +25,8 @@ import software.amazon.kinesis.annotations.KinesisClientInternalApi;
@Slf4j
@KinesisClientInternalApi
class RejectedTaskEvent implements DiagnosticEvent {
private static final String MESSAGE = "Review your thread configuration to prevent task rejections. " +
"Task rejections will slow down your application and some shards may stop processing. ";
private static final String MESSAGE = "Review your thread configuration to prevent task rejections. "
+ "Task rejections will slow down your application and some shards may stop processing. ";
private ExecutorStateEvent executorStateEvent;
private Throwable throwable;

336
amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/Scheduler.java
View file

@ -15,10 +15,6 @@
package software.amazon.kinesis.coordinator;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Stopwatch;
import io.reactivex.rxjava3.plugins.RxJavaPlugins;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
@ -45,6 +41,10 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.stream.Collectors;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Stopwatch;
import io.reactivex.rxjava3.plugins.RxJavaPlugins;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.NoArgsConstructor;
@ -188,6 +188,7 @@ public class Scheduler implements Runnable {
private final Stopwatch streamSyncWatch = Stopwatch.createUnstarted();
private boolean leasesSyncedOnAppInit = false;
@Getter(AccessLevel.NONE)
private final AtomicBoolean leaderSynced = new AtomicBoolean(false);
@ -206,15 +207,23 @@ public class Scheduler implements Runnable {
@VisibleForTesting
protected boolean gracefuleShutdownStarted = false;
public Scheduler(@NonNull final CheckpointConfig checkpointConfig,
@NonNull final CoordinatorConfig coordinatorConfig,
@NonNull final LeaseManagementConfig leaseManagementConfig,
@NonNull final LifecycleConfig lifecycleConfig,
@NonNull final MetricsConfig metricsConfig,
@NonNull final ProcessorConfig processorConfig,
@NonNull final RetrievalConfig retrievalConfig) {
this(checkpointConfig, coordinatorConfig, leaseManagementConfig, lifecycleConfig, metricsConfig,
processorConfig, retrievalConfig, new DiagnosticEventFactory());
public Scheduler(
@NonNull final CheckpointConfig checkpointConfig,
@NonNull final CoordinatorConfig coordinatorConfig,
@NonNull final LeaseManagementConfig leaseManagementConfig,
@NonNull final LifecycleConfig lifecycleConfig,
@NonNull final MetricsConfig metricsConfig,
@NonNull final ProcessorConfig processorConfig,
@NonNull final RetrievalConfig retrievalConfig) {
this(
checkpointConfig,
coordinatorConfig,
leaseManagementConfig,
lifecycleConfig,
metricsConfig,
processorConfig,
retrievalConfig,
new DiagnosticEventFactory());
}
/**
@ -222,14 +231,15 @@ public class Scheduler implements Runnable {
* is desired for testing. This constructor is only used for testing to provide a mock DiagnosticEventFactory.
*/
@VisibleForTesting
protected Scheduler(@NonNull final CheckpointConfig checkpointConfig,
@NonNull final CoordinatorConfig coordinatorConfig,
@NonNull final LeaseManagementConfig leaseManagementConfig,
@NonNull final LifecycleConfig lifecycleConfig,
@NonNull final MetricsConfig metricsConfig,
@NonNull final ProcessorConfig processorConfig,
@NonNull final RetrievalConfig retrievalConfig,
@NonNull final DiagnosticEventFactory diagnosticEventFactory) {
protected Scheduler(
@NonNull final CheckpointConfig checkpointConfig,
@NonNull final CoordinatorConfig coordinatorConfig,
@NonNull final LeaseManagementConfig leaseManagementConfig,
@NonNull final LifecycleConfig lifecycleConfig,
@NonNull final MetricsConfig metricsConfig,
@NonNull final ProcessorConfig processorConfig,
@NonNull final RetrievalConfig retrievalConfig,
@NonNull final DiagnosticEventFactory diagnosticEventFactory) {
this.checkpointConfig = checkpointConfig;
this.coordinatorConfig = coordinatorConfig;
this.leaseManagementConfig = leaseManagementConfig;
@ -242,16 +252,14 @@ public class Scheduler implements Runnable {
this.streamTracker = retrievalConfig.streamTracker();
this.isMultiStreamMode = streamTracker.isMultiStream();
this.formerStreamsLeasesDeletionStrategy = streamTracker.formerStreamsLeasesDeletionStrategy();
streamTracker.streamConfigList().forEach(
sc -> currentStreamConfigMap.put(sc.streamIdentifier(), sc));
streamTracker.streamConfigList().forEach(sc -> currentStreamConfigMap.put(sc.streamIdentifier(), sc));
log.info("Initial state: {}", currentStreamConfigMap.values());
this.maxInitializationAttempts = this.coordinatorConfig.maxInitializationAttempts();
this.metricsFactory = this.metricsConfig.metricsFactory();
// Determine leaseSerializer based on availability of MultiStreamTracker.
final LeaseSerializer leaseSerializer = isMultiStreamMode ?
new DynamoDBMultiStreamLeaseSerializer() :
new DynamoDBLeaseSerializer();
final LeaseSerializer leaseSerializer =
isMultiStreamMode ? new DynamoDBMultiStreamLeaseSerializer() : new DynamoDBLeaseSerializer();
this.leaseCoordinator = this.leaseManagementConfig
.leaseManagementFactory(leaseSerializer, isMultiStreamMode)
.createLeaseCoordinator(this.metricsFactory);
@ -260,8 +268,9 @@ public class Scheduler implements Runnable {
//
// TODO: Figure out what to do with lease manage <=> checkpoint relationship
//
this.checkpoint = this.checkpointConfig.checkpointFactory().createCheckpointer(this.leaseCoordinator,
this.leaseRefresher);
this.checkpoint = this.checkpointConfig
.checkpointFactory()
.createCheckpointer(this.leaseCoordinator, this.leaseRefresher);
//
// TODO: Move this configuration to lifecycle
@ -282,38 +291,46 @@ public class Scheduler implements Runnable {
if (coordinatorConfig.gracefulShutdownCoordinator() != null) {
this.gracefulShutdownCoordinator = coordinatorConfig.gracefulShutdownCoordinator();
} else {
this.gracefulShutdownCoordinator = this.coordinatorConfig.coordinatorFactory()
.createGracefulShutdownCoordinator();
this.gracefulShutdownCoordinator =
this.coordinatorConfig.coordinatorFactory().createGracefulShutdownCoordinator();
}
if (coordinatorConfig.workerStateChangeListener() != null) {
this.workerStateChangeListener = coordinatorConfig.workerStateChangeListener();
} else {
this.workerStateChangeListener = this.coordinatorConfig.coordinatorFactory()
.createWorkerStateChangeListener();
this.workerStateChangeListener =
this.coordinatorConfig.coordinatorFactory().createWorkerStateChangeListener();
}
this.leaderDecider = new DeterministicShuffleShardSyncLeaderDecider(leaseRefresher,
Executors.newSingleThreadScheduledExecutor(),
PERIODIC_SHARD_SYNC_MAX_WORKERS_DEFAULT);
this.leaderDecider = new DeterministicShuffleShardSyncLeaderDecider(
leaseRefresher, Executors.newSingleThreadScheduledExecutor(), PERIODIC_SHARD_SYNC_MAX_WORKERS_DEFAULT);
this.failoverTimeMillis = this.leaseManagementConfig.failoverTimeMillis();
this.taskBackoffTimeMillis = this.lifecycleConfig.taskBackoffTimeMillis();
this.listShardsBackoffTimeMillis = this.retrievalConfig.listShardsBackoffTimeInMillis();
this.maxListShardsRetryAttempts = this.retrievalConfig.maxListShardsRetryAttempts();
this.shardDetectorProvider = streamConfig -> createOrGetShardSyncTaskManager(streamConfig).shardDetector();
this.shardDetectorProvider =
streamConfig -> createOrGetShardSyncTaskManager(streamConfig).shardDetector();
this.ignoreUnexpetedChildShards = this.leaseManagementConfig.ignoreUnexpectedChildShards();
this.aggregatorUtil = this.lifecycleConfig.aggregatorUtil();
this.hierarchicalShardSyncerProvider = streamConfig -> createOrGetShardSyncTaskManager(streamConfig).hierarchicalShardSyncer();
this.schedulerInitializationBackoffTimeMillis = this.coordinatorConfig.schedulerInitializationBackoffTimeMillis();
this.hierarchicalShardSyncerProvider =
streamConfig -> createOrGetShardSyncTaskManager(streamConfig).hierarchicalShardSyncer();
this.schedulerInitializationBackoffTimeMillis =
this.coordinatorConfig.schedulerInitializationBackoffTimeMillis();
this.leaderElectedPeriodicShardSyncManager = new PeriodicShardSyncManager(
leaseManagementConfig.workerIdentifier(), leaderDecider, leaseRefresher, currentStreamConfigMap,
shardSyncTaskManagerProvider, streamToShardSyncTaskManagerMap, isMultiStreamMode, metricsFactory,
leaseManagementConfig.workerIdentifier(),
leaderDecider,
leaseRefresher,
currentStreamConfigMap,
shardSyncTaskManagerProvider,
streamToShardSyncTaskManagerMap,
isMultiStreamMode,
metricsFactory,
leaseManagementConfig.leasesRecoveryAuditorExecutionFrequencyMillis(),
leaseManagementConfig.leasesRecoveryAuditorInconsistencyConfidenceThreshold(),
leaderSynced);
this.leaseCleanupManager = this.leaseManagementConfig.leaseManagementFactory(leaseSerializer, isMultiStreamMode)
this.leaseCleanupManager = this.leaseManagementConfig
.leaseManagementFactory(leaseSerializer, isMultiStreamMode)
.createLeaseCleanupManager(metricsFactory);
this.schemaRegistryDecoder =
this.retrievalConfig.glueSchemaRegistryDeserializer() == null ?
null
this.schemaRegistryDecoder = this.retrievalConfig.glueSchemaRegistryDeserializer() == null
? null
: new SchemaRegistryDecoder(this.retrievalConfig.glueSchemaRegistryDeserializer());
}
@ -357,9 +374,10 @@ public class Scheduler implements Runnable {
if (!skipShardSyncAtWorkerInitializationIfLeasesExist || leaseRefresher.isLeaseTableEmpty()) {
if (shouldInitiateLeaseSync()) {
log.info("Worker {} is initiating the lease sync.", leaseManagementConfig.workerIdentifier());
log.info(
"Worker {} is initiating the lease sync.",
leaseManagementConfig.workerIdentifier());
leaderElectedPeriodicShardSyncManager.syncShardsOnce();
}
} else {
log.info("Skipping shard sync per configuration setting (and lease table is not empty)");
@ -402,13 +420,15 @@ public class Scheduler implements Runnable {
}
@VisibleForTesting
boolean shouldInitiateLeaseSync() throws InterruptedException,
DependencyException, ProvisionedThroughputException, InvalidStateException {
long waitTime = ThreadLocalRandom.current().nextLong(MIN_WAIT_TIME_FOR_LEASE_TABLE_CHECK_MILLIS, MAX_WAIT_TIME_FOR_LEASE_TABLE_CHECK_MILLIS);
boolean shouldInitiateLeaseSync()
throws InterruptedException, DependencyException, ProvisionedThroughputException, InvalidStateException {
long waitTime = ThreadLocalRandom.current()
.nextLong(MIN_WAIT_TIME_FOR_LEASE_TABLE_CHECK_MILLIS, MAX_WAIT_TIME_FOR_LEASE_TABLE_CHECK_MILLIS);
long waitUntil = System.currentTimeMillis() + waitTime;
boolean shouldInitiateLeaseSync = true;
while (System.currentTimeMillis() < waitUntil && (shouldInitiateLeaseSync = leaseRefresher.isLeaseTableEmpty())) {
while (System.currentTimeMillis() < waitUntil
&& (shouldInitiateLeaseSync = leaseRefresher.isLeaseTableEmpty())) {
// check every 3 seconds if lease table is still empty,
// to minimize contention between all workers bootstrapping at the same time
log.info("Lease table is still empty. Checking again in {} ms", LEASE_TABLE_CHECK_FREQUENCY_MILLIS);
@ -422,8 +442,8 @@ public class Scheduler implements Runnable {
try {
Set<ShardInfo> assignedShards = new HashSet<>();
for (ShardInfo shardInfo : getShardInfoForAssignments()) {
ShardConsumer shardConsumer = createOrGetShardConsumer(shardInfo,
processorConfig.shardRecordProcessorFactory(), leaseCleanupManager);
ShardConsumer shardConsumer = createOrGetShardConsumer(
shardInfo, processorConfig.shardRecordProcessorFactory(), leaseCleanupManager);
shardConsumer.executeLifecycle();
assignedShards.add(shardInfo);
@ -444,8 +464,10 @@ public class Scheduler implements Runnable {
slog.info("Sleeping ...");
Thread.sleep(shardConsumerDispatchPollIntervalMillis);
} catch (Exception e) {
log.error("Worker.run caught exception, sleeping for {} milli seconds!",
shardConsumerDispatchPollIntervalMillis, e);
log.error(
"Worker.run caught exception, sleeping for {} milli seconds!",
shardConsumerDispatchPollIntervalMillis,
e);
try {
Thread.sleep(shardConsumerDispatchPollIntervalMillis);
} catch (InterruptedException ex) {
@ -470,11 +492,12 @@ public class Scheduler implements Runnable {
final Set<StreamIdentifier> streamsSynced = new HashSet<>();
if (shouldSyncStreamsNow()) {
final MetricsScope metricsScope = MetricsUtil.createMetricsWithOperation(metricsFactory, MULTI_STREAM_TRACKER);
final MetricsScope metricsScope =
MetricsUtil.createMetricsWithOperation(metricsFactory, MULTI_STREAM_TRACKER);
try {
final Map<StreamIdentifier, StreamConfig> newStreamConfigMap = streamTracker.streamConfigList()
.stream().collect(Collectors.toMap(StreamConfig::streamIdentifier, Function.identity()));
final Map<StreamIdentifier, StreamConfig> newStreamConfigMap = streamTracker.streamConfigList().stream()
.collect(Collectors.toMap(StreamConfig::streamIdentifier, Function.identity()));
// This is done to ensure that we clean up the stale streams lingering in the lease table.
// Only sync from lease table again if the currentStreamConfigMap and newStreamConfigMap contain
// different set of streams and Leader has not synced the leases yet
@ -487,12 +510,16 @@ public class Scheduler implements Runnable {
final Set<StreamIdentifier> streamsFromLeaseTable = leaseTableLeases.stream()
.map(lease -> StreamIdentifier.multiStreamInstance(lease.streamIdentifier()))
.collect(Collectors.toSet());
// Remove stream from currentStreamConfigMap if this stream in not in the lease table and newStreamConfigMap.
// Remove stream from currentStreamConfigMap if this stream in not in the lease table and
// newStreamConfigMap.
// This means that the leases have already been deleted by the last leader.
currentStreamConfigMap.keySet().stream()
.filter(streamIdentifier -> !newStreamConfigMap.containsKey(streamIdentifier)
&& !streamsFromLeaseTable.contains(streamIdentifier)).forEach(stream -> {
log.info("Removing stream {} from currentStreamConfigMap due to not being active", stream);
&& !streamsFromLeaseTable.contains(streamIdentifier))
.forEach(stream -> {
log.info(
"Removing stream {} from currentStreamConfigMap due to not being active",
stream);
currentStreamConfigMap.remove(stream);
staleStreamDeletionMap.remove(stream);
streamsSynced.add(stream);
@ -519,18 +546,21 @@ public class Scheduler implements Runnable {
staleStreamDeletionMap.putIfAbsent(streamIdentifier, Instant.now());
}
};
if (formerStreamsLeasesDeletionStrategy.leaseDeletionType() == FORMER_STREAMS_AUTO_DETECTION_DEFERRED_DELETION) {
if (formerStreamsLeasesDeletionStrategy.leaseDeletionType()
== FORMER_STREAMS_AUTO_DETECTION_DEFERRED_DELETION) {
// Now, we are identifying the stale/old streams and enqueuing it for deferred deletion.
// It is assumed that all the workers will always have the latest and consistent snapshot of streams
// from the multiStreamTracker.
//
// The following streams transition state among two workers are NOT considered safe, where Worker 2, on
// The following streams transition state among two workers are NOT considered safe, where Worker 2,
// on
// initialization learn about D from lease table and delete the leases for D, as it is not available
// in its latest MultiStreamTracker.
// Worker 1 : A,B,C -> A,B,C,D (latest)
// Worker 2 : BOOTS_UP -> A,B,C (stale)
//
// The following streams transition state among two workers are NOT considered safe, where Worker 2 might
// The following streams transition state among two workers are NOT considered safe, where Worker 2
// might
// end up deleting the leases for A and D and lose progress made so far.
// Worker 1 : A,B,C -> A,B,C,D (latest)
// Worker 2 : A,B,C -> B,C (stale/partial)
@ -539,13 +569,16 @@ public class Scheduler implements Runnable {
// before attempting to delete it, we will be deferring the leases deletion based on the
// defer time period.
currentStreamConfigMap.keySet().forEach(enqueueStreamLeaseDeletionOperation);
} else if (formerStreamsLeasesDeletionStrategy.leaseDeletionType() == StreamsLeasesDeletionType.PROVIDED_STREAMS_DEFERRED_DELETION) {
Optional.ofNullable(formerStreamsLeasesDeletionStrategy.streamIdentifiersForLeaseCleanup()).ifPresent(
streamIdentifiers -> streamIdentifiers.forEach(enqueueStreamLeaseDeletionOperation));
} else if (formerStreamsLeasesDeletionStrategy.leaseDeletionType()
== StreamsLeasesDeletionType.PROVIDED_STREAMS_DEFERRED_DELETION) {
Optional.ofNullable(formerStreamsLeasesDeletionStrategy.streamIdentifiersForLeaseCleanup())
.ifPresent(streamIdentifiers ->
streamIdentifiers.forEach(enqueueStreamLeaseDeletionOperation));
} else {
// Remove the old/stale streams identified through the new and existing streams list, without
// cleaning up their leases. Disabling deprecated shard sync + lease cleanup through a flag.
Iterator<StreamIdentifier> currentSetOfStreamsIter = currentStreamConfigMap.keySet().iterator();
Iterator<StreamIdentifier> currentSetOfStreamsIter =
currentStreamConfigMap.keySet().iterator();
while (currentSetOfStreamsIter.hasNext()) {
StreamIdentifier streamIdentifier = currentSetOfStreamsIter.next();
if (!newStreamConfigMap.containsKey(streamIdentifier)) {
@ -553,13 +586,14 @@ public class Scheduler implements Runnable {
log.info(
"Found old/deleted stream : {}. Triggering shard sync. Removing from tracked active streams.",
streamIdentifier);
ShardSyncTaskManager shardSyncTaskManager = createOrGetShardSyncTaskManager(
currentStreamConfigMap.get(streamIdentifier));
ShardSyncTaskManager shardSyncTaskManager =
createOrGetShardSyncTaskManager(currentStreamConfigMap.get(streamIdentifier));
shardSyncTaskManager.submitShardSyncTask();
} else {
log.info(
"Found old/deleted stream : {}. Removing from tracked active streams, but not cleaning up leases,"
+ " as part of this workflow", streamIdentifier);
+ " as part of this workflow",
streamIdentifier);
}
currentSetOfStreamsIter.remove();
streamsSynced.add(streamIdentifier);
@ -573,22 +607,24 @@ public class Scheduler implements Runnable {
// StreamIdentifiers are eligible for deletion only when the deferment period has elapsed and
// the streamIdentifiersForLeaseCleanup are not present in the latest snapshot.
final Map<Boolean, Set<StreamIdentifier>> staleStreamIdDeletionDecisionMap =
staleStreamDeletionMap.keySet().stream().collect(
Collectors.partitioningBy(newStreamConfigMap::containsKey, Collectors.toSet()));
final Set<StreamIdentifier> staleStreamIdsToBeDeleted = staleStreamIdDeletionDecisionMap.get(false)
.stream().filter(streamIdentifier ->
Duration.between(staleStreamDeletionMap.get(streamIdentifier), Instant.now())
.toMillis() >= waitPeriodToDeleteOldStreams.toMillis())
.collect(Collectors.toSet());
staleStreamDeletionMap.keySet().stream()
.collect(
Collectors.partitioningBy(newStreamConfigMap::containsKey, Collectors.toSet()));
final Set<StreamIdentifier> staleStreamIdsToBeDeleted =
staleStreamIdDeletionDecisionMap.get(false).stream()
.filter(streamIdentifier ->
Duration.between(staleStreamDeletionMap.get(streamIdentifier), Instant.now())
.toMillis()
>= waitPeriodToDeleteOldStreams.toMillis())
.collect(Collectors.toSet());
// These are the streams which are deleted in Kinesis and we encounter resource not found during
// shardSyncTask. This is applicable in MultiStreamMode only, in case of SingleStreamMode, store will
// not have any data.
// Filter streams based on newStreamConfigMap so that we don't override input to KCL in any case.
final Set<StreamIdentifier> deletedStreamSet = this.deletedStreamListProvider
.purgeAllDeletedStream()
.stream()
.filter(streamIdentifier -> !newStreamConfigMap.containsKey(streamIdentifier))
.collect(Collectors.toSet());
final Set<StreamIdentifier> deletedStreamSet =
this.deletedStreamListProvider.purgeAllDeletedStream().stream()
.filter(streamIdentifier -> !newStreamConfigMap.containsKey(streamIdentifier))
.collect(Collectors.toSet());
if (deletedStreamSet.size() > 0) {
log.info("Stale streams to delete: {}", deletedStreamSet);
staleStreamIdsToBeDeleted.addAll(deletedStreamSet);
@ -603,18 +639,24 @@ public class Scheduler implements Runnable {
if (!staleStreamDeletionMap.isEmpty()) {
log.warn(
"Streams enqueued for deletion for lease table cleanup along with their scheduled time for deletion: {} ",
staleStreamDeletionMap.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey,
entry -> entry.getValue().plus(waitPeriodToDeleteOldStreams))));
staleStreamDeletionMap.entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue()
.plus(waitPeriodToDeleteOldStreams))));
}
streamSyncWatch.reset().start();
MetricsUtil.addCount(metricsScope, ACTIVE_STREAMS_COUNT, newStreamConfigMap.size(), MetricsLevel.SUMMARY);
MetricsUtil.addCount(metricsScope, PENDING_STREAMS_DELETION_COUNT, staleStreamDeletionMap.size(),
MetricsUtil.addCount(
metricsScope, ACTIVE_STREAMS_COUNT, newStreamConfigMap.size(), MetricsLevel.SUMMARY);
MetricsUtil.addCount(
metricsScope,
PENDING_STREAMS_DELETION_COUNT,
staleStreamDeletionMap.size(),
MetricsLevel.SUMMARY);
MetricsUtil.addCount(metricsScope, NON_EXISTING_STREAM_DELETE_COUNT, deletedStreamSet.size(),
MetricsLevel.SUMMARY);
MetricsUtil.addCount(metricsScope, DELETED_STREAMS_COUNT, deletedStreamsLeases.size(), MetricsLevel.SUMMARY);
MetricsUtil.addCount(
metricsScope, NON_EXISTING_STREAM_DELETE_COUNT, deletedStreamSet.size(), MetricsLevel.SUMMARY);
MetricsUtil.addCount(
metricsScope, DELETED_STREAMS_COUNT, deletedStreamsLeases.size(), MetricsLevel.SUMMARY);
} finally {
MetricsUtil.endScope(metricsScope);
}
@ -624,8 +666,7 @@ public class Scheduler implements Runnable {
@VisibleForTesting
boolean shouldSyncStreamsNow() {
return isMultiStreamMode &&
(streamSyncWatch.elapsed(TimeUnit.MILLISECONDS) > NEW_STREAM_CHECK_INTERVAL_MILLIS);
return isMultiStreamMode && (streamSyncWatch.elapsed(TimeUnit.MILLISECONDS) > NEW_STREAM_CHECK_INTERVAL_MILLIS);
}
@VisibleForTesting
@ -642,7 +683,8 @@ public class Scheduler implements Runnable {
private List<MultiStreamLease> fetchMultiStreamLeases()
throws DependencyException, ProvisionedThroughputException, InvalidStateException {
return (List<MultiStreamLease>) ((List) leaseCoordinator.leaseRefresher().listLeases());
return (List<MultiStreamLease>)
((List) leaseCoordinator.leaseRefresher().listLeases());
}
private void removeStreamsFromStaleStreamsList(Set<StreamIdentifier> streamIdentifiers) {
@ -659,8 +701,9 @@ public class Scheduler implements Runnable {
log.info("Deleting streams: {}", streamIdentifiers);
final Set<StreamIdentifier> streamsSynced = new HashSet<>();
final List<MultiStreamLease> leases = fetchMultiStreamLeases();
final Map<String, List<MultiStreamLease>> streamIdToShardsMap = leases.stream().collect(
Collectors.groupingBy(MultiStreamLease::streamIdentifier, Collectors.toCollection(ArrayList::new)));
final Map<String, List<MultiStreamLease>> streamIdToShardsMap = leases.stream()
.collect(Collectors.groupingBy(
MultiStreamLease::streamIdentifier, Collectors.toCollection(ArrayList::new)));
for (StreamIdentifier streamIdentifier : streamIdentifiers) {
log.warn("Found old/deleted stream: {}. Directly deleting leases of this stream.", streamIdentifier);
// Removing streamIdentifier from this map so PSSM doesn't think there is a hole in the stream while
@ -690,7 +733,8 @@ public class Scheduler implements Runnable {
} catch (DependencyException | InvalidStateException | ProvisionedThroughputException e) {
log.error(
"Unable to delete stale stream lease {}. Skipping further deletions for this stream. Will retry later.",
lease.leaseKey(), e);
lease.leaseKey(),
e);
return false;
}
}
@ -760,8 +804,8 @@ public class Scheduler implements Runnable {
public CompletableFuture<Boolean> startGracefulShutdown() {
synchronized (this) {
if (gracefulShutdownFuture == null) {
gracefulShutdownFuture = gracefulShutdownCoordinator
.startGracefulShutdown(createGracefulShutdownCallable());
gracefulShutdownFuture =
gracefulShutdownCoordinator.startGracefulShutdown(createGracefulShutdownCallable());
}
}
return gracefulShutdownFuture;
@ -809,13 +853,15 @@ public class Scheduler implements Runnable {
// If there are no leases notification is already completed, but we still need to shutdown the worker.
//
this.shutdown();
return GracefulShutdownContext.builder().finalShutdownLatch(finalShutdownLatch).build();
return GracefulShutdownContext.builder()
.finalShutdownLatch(finalShutdownLatch)
.build();
}
CountDownLatch shutdownCompleteLatch = new CountDownLatch(leases.size());
CountDownLatch notificationCompleteLatch = new CountDownLatch(leases.size());
for (Lease lease : leases) {
ShutdownNotification shutdownNotification = new ShardConsumerShutdownNotification(leaseCoordinator,
lease, notificationCompleteLatch, shutdownCompleteLatch);
ShutdownNotification shutdownNotification = new ShardConsumerShutdownNotification(
leaseCoordinator, lease, notificationCompleteLatch, shutdownCompleteLatch);
ShardInfo shardInfo = DynamoDBLeaseCoordinator.convertLeaseToAssignment(lease);
ShardConsumer consumer = shardInfoShardConsumerMap.get(shardInfo);
if (consumer != null) {
@ -929,9 +975,10 @@ public class Scheduler implements Runnable {
* Kinesis shard info
* @return ShardConsumer for the shard
*/
ShardConsumer createOrGetShardConsumer(@NonNull final ShardInfo shardInfo,
@NonNull final ShardRecordProcessorFactory shardRecordProcessorFactory,
@NonNull final LeaseCleanupManager leaseCleanupManager) {
ShardConsumer createOrGetShardConsumer(
@NonNull final ShardInfo shardInfo,
@NonNull final ShardRecordProcessorFactory shardRecordProcessorFactory,
@NonNull final LeaseCleanupManager leaseCleanupManager) {
ShardConsumer consumer = shardInfoShardConsumerMap.get(shardInfo);
// Instantiate a new consumer if we don't have one, or the one we
// had was from an earlier
@ -948,20 +995,23 @@ public class Scheduler implements Runnable {
}
private ShardSyncTaskManager createOrGetShardSyncTaskManager(StreamConfig streamConfig) {
return streamToShardSyncTaskManagerMap.computeIfAbsent(streamConfig, s -> shardSyncTaskManagerProvider.apply(s));
return streamToShardSyncTaskManagerMap.computeIfAbsent(
streamConfig, s -> shardSyncTaskManagerProvider.apply(s));
}
protected ShardConsumer buildConsumer(@NonNull final ShardInfo shardInfo,
@NonNull final ShardRecordProcessorFactory shardRecordProcessorFactory,
@NonNull final LeaseCleanupManager leaseCleanupManager) {
ShardRecordProcessorCheckpointer checkpointer = coordinatorConfig.coordinatorFactory().createRecordProcessorCheckpointer(shardInfo,
checkpoint);
protected ShardConsumer buildConsumer(
@NonNull final ShardInfo shardInfo,
@NonNull final ShardRecordProcessorFactory shardRecordProcessorFactory,
@NonNull final LeaseCleanupManager leaseCleanupManager) {
ShardRecordProcessorCheckpointer checkpointer =
coordinatorConfig.coordinatorFactory().createRecordProcessorCheckpointer(shardInfo, checkpoint);
// The only case where streamName is not available will be when multistreamtracker not set. In this case,
// get the default stream name for the single stream application.
final StreamIdentifier streamIdentifier = getStreamIdentifier(shardInfo.streamIdentifierSerOpt());
// Irrespective of single stream app or multi stream app, streamConfig should always be available.
// If we have a shardInfo, that is not present in currentStreamConfigMap for whatever reason, then return default stream config
// If we have a shardInfo, that is not present in currentStreamConfigMap for whatever reason, then return
// default stream config
// to gracefully complete the reading.
StreamConfig streamConfig = currentStreamConfigMap.get(streamIdentifier);
if (streamConfig == null) {
@ -969,8 +1019,10 @@ public class Scheduler implements Runnable {
log.info("Created orphan {}", streamConfig);
}
Validate.notNull(streamConfig, "StreamConfig should not be null");
RecordsPublisher cache = retrievalConfig.retrievalFactory().createGetRecordsCache(shardInfo, streamConfig, metricsFactory);
ShardConsumerArgument argument = new ShardConsumerArgument(shardInfo,
RecordsPublisher cache =
retrievalConfig.retrievalFactory().createGetRecordsCache(shardInfo, streamConfig, metricsFactory);
ShardConsumerArgument argument = new ShardConsumerArgument(
shardInfo,
streamConfig.streamIdentifier(),
leaseCoordinator,
executorService,
@ -993,10 +1045,15 @@ public class Scheduler implements Runnable {
hierarchicalShardSyncerProvider.apply(streamConfig),
metricsFactory,
leaseCleanupManager,
schemaRegistryDecoder
);
return new ShardConsumer(cache, executorService, shardInfo, lifecycleConfig.logWarningForTaskAfterMillis(),
argument, lifecycleConfig.taskExecutionListener(), lifecycleConfig.readTimeoutsToIgnoreBeforeWarning());
schemaRegistryDecoder);
return new ShardConsumer(
cache,
executorService,
shardInfo,
lifecycleConfig.logWarningForTaskAfterMillis(),
argument,
lifecycleConfig.taskExecutionListener(),
lifecycleConfig.readTimeoutsToIgnoreBeforeWarning());
}
/**
@ -1030,16 +1087,16 @@ public class Scheduler implements Runnable {
*/
private void registerErrorHandlerForUndeliverableAsyncTaskExceptions() {
RxJavaPlugins.setErrorHandler(t -> {
ExecutorStateEvent executorStateEvent = diagnosticEventFactory.executorStateEvent(executorService,
leaseCoordinator);
ExecutorStateEvent executorStateEvent =
diagnosticEventFactory.executorStateEvent(executorService, leaseCoordinator);
RejectedTaskEvent rejectedTaskEvent = diagnosticEventFactory.rejectedTaskEvent(executorStateEvent, t);
rejectedTaskEvent.accept(diagnosticEventHandler);
});
}
private void logExecutorState() {
ExecutorStateEvent executorStateEvent = diagnosticEventFactory.executorStateEvent(executorService,
leaseCoordinator);
ExecutorStateEvent executorStateEvent =
diagnosticEventFactory.executorStateEvent(executorService, leaseCoordinator);
executorStateEvent.accept(diagnosticEventHandler);
}
@ -1049,7 +1106,8 @@ public class Scheduler implements Runnable {
streamIdentifier = StreamIdentifier.multiStreamInstance(streamIdentifierString.get());
} else {
Validate.isTrue(!isMultiStreamMode, "Should not be in MultiStream Mode");
streamIdentifier = this.currentStreamConfigMap.values().iterator().next().streamIdentifier();
streamIdentifier =
this.currentStreamConfigMap.values().iterator().next().streamIdentifier();
}
Validate.notNull(streamIdentifier, "Stream identifier should not be empty");
return streamIdentifier;
@ -1070,13 +1128,16 @@ public class Scheduler implements Runnable {
*/
private static StreamConfig withStreamArn(
@NonNull final StreamConfig streamConfig, @NonNull final Region kinesisRegion) {
Validate.isTrue(streamConfig.streamIdentifier().accountIdOptional().isPresent(),
"accountId should not be empty");
Validate.isTrue(streamConfig.streamIdentifier().streamCreationEpochOptional().isPresent(),
Validate.isTrue(
streamConfig.streamIdentifier().accountIdOptional().isPresent(), "accountId should not be empty");
Validate.isTrue(
streamConfig.streamIdentifier().streamCreationEpochOptional().isPresent(),
"streamCreationEpoch should not be empty");
log.info("Constructing stream ARN for {} using the Kinesis client's configured region - {}.",
streamConfig.streamIdentifier(), kinesisRegion);
log.info(
"Constructing stream ARN for {} using the Kinesis client's configured region - {}.",
streamConfig.streamIdentifier(),
kinesisRegion);
final StreamIdentifier streamIdentifierWithArn = StreamIdentifier.multiStreamInstance(
constructStreamArn(
@ -1106,13 +1167,22 @@ public class Scheduler implements Runnable {
@NonNull final StreamIdentifier streamIdentifier, @NonNull final StreamConfig streamConfig) {
final Region kinesisRegion = getKinesisRegion();
return super.put(streamIdentifier, streamConfig.streamIdentifier().streamArnOptional()
.map(streamArn -> {
Validate.isTrue(kinesisRegion.id().equals(streamArn.region().get()),
"The provided streamARN " + streamArn
+ " does not match the Kinesis client's configured region - " + kinesisRegion);
return streamConfig;
}).orElse(isMultiStreamMode ? withStreamArn(streamConfig, kinesisRegion) : streamConfig));
return super.put(
streamIdentifier,
streamConfig
.streamIdentifier()
.streamArnOptional()
.map(streamArn -> {
Validate.isTrue(
kinesisRegion
.id()
.equals(streamArn.region().get()),
"The provided streamARN " + streamArn
+ " does not match the Kinesis client's configured region - "
+ kinesisRegion);
return streamConfig;
})
.orElse(isMultiStreamMode ? withStreamArn(streamConfig, kinesisRegion) : streamConfig));
}
}

14
amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/SchedulerCoordinatorFactory.java
View file

@ -22,7 +22,6 @@ import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import lombok.Data;
import lombok.NonNull;
import software.amazon.kinesis.annotations.KinesisClientInternalApi;
@ -41,15 +40,16 @@ public class SchedulerCoordinatorFactory implements CoordinatorFactory {
*/
@Override
public ExecutorService createExecutorService() {
return new SchedulerThreadPoolExecutor(
new ThreadFactoryBuilder().setNameFormat("ShardRecordProcessor-%04d").build());
return new SchedulerThreadPoolExecutor(new ThreadFactoryBuilder()
.setNameFormat("ShardRecordProcessor-%04d")
.build());
}
static class SchedulerThreadPoolExecutor extends ThreadPoolExecutor {
private static final long DEFAULT_KEEP_ALIVE = 60L;
SchedulerThreadPoolExecutor(ThreadFactory threadFactory) {
super(0, Integer.MAX_VALUE, DEFAULT_KEEP_ALIVE, TimeUnit.SECONDS, new SynchronousQueue<>(),
threadFactory);
super(0, Integer.MAX_VALUE, DEFAULT_KEEP_ALIVE, TimeUnit.SECONDS, new SynchronousQueue<>(), threadFactory);
}
}
@ -57,8 +57,8 @@ public class SchedulerCoordinatorFactory implements CoordinatorFactory {
* {@inheritDoc}
*/
@Override
public ShardRecordProcessorCheckpointer createRecordProcessorCheckpointer(@NonNull final ShardInfo shardInfo,
@NonNull final Checkpointer checkpoint) {
public ShardRecordProcessorCheckpointer createRecordProcessorCheckpointer(
@NonNull final ShardInfo shardInfo, @NonNull final Checkpointer checkpoint) {
return new ShardRecordProcessorCheckpointer(shardInfo, checkpoint);
}
}

3
amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/WorkerStateChangeListener.java
View file

@ -29,6 +29,5 @@ public interface WorkerStateChangeListener {
void onWorkerStateChange(WorkerState newState);
default void onAllInitializationAttemptsFailed(Throwable e) {
}
default void onAllInitializationAttemptsFailed(Throwable e) {}
}

1
amazon-kinesis-client/src/main/java/software/amazon/kinesis/exceptions/InvalidStateException.java
View file

@ -36,5 +36,4 @@ public class InvalidStateException extends KinesisClientLibNonRetryableException
public InvalidStateException(String message, Exception e) {
super(message, e);
}
}

1
amazon-kinesis-client/src/main/java/software/amazon/kinesis/exceptions/KinesisClientLibDependencyException.java
View file

@ -37,5 +37,4 @@ public class KinesisClientLibDependencyException extends KinesisClientLibRetryab
public KinesisClientLibDependencyException(String message, Exception e) {
super(message, e);
}
}

1
amazon-kinesis-client/src/main/java/software/amazon/kinesis/exceptions/KinesisClientLibException.java
View file

@ -41,5 +41,4 @@ public abstract class KinesisClientLibException extends Exception {
public KinesisClientLibException(String message, Throwable cause) {
super(message, cause);
}
}

1
amazon-kinesis-client/src/main/java/software/amazon/kinesis/exceptions/ShutdownException.java
View file

@ -35,5 +35,4 @@ public class ShutdownException extends KinesisClientLibNonRetryableException {
public ShutdownException(String message, Exception e) {
super(message, e);
}
}

1
amazon-kinesis-client/src/main/java/software/amazon/kinesis/exceptions/ThrottlingException.java
View file

@ -35,5 +35,4 @@ public class ThrottlingException extends KinesisClientLibRetryableException {
public ThrottlingException(String message, Exception e) {
super(message, e);
}
}

1
amazon-kinesis-client/src/main/java/software/amazon/kinesis/exceptions/internal/BlockedOnParentShardException.java
View file

@ -43,5 +43,4 @@ public class BlockedOnParentShardException extends KinesisClientLibRetryableExce
public BlockedOnParentShardException(String message, Exception e) {
super(message, e);
}
}

13
amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/DynamoUtils.java
View file

@ -14,15 +14,15 @@
*/
package software.amazon.kinesis.leases;
import software.amazon.awssdk.core.SdkBytes;
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
import software.amazon.kinesis.annotations.KinesisClientInternalApi;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import software.amazon.awssdk.core.SdkBytes;
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
import software.amazon.kinesis.annotations.KinesisClientInternalApi;
/**
* Static utility functions used by our LeaseSerializers.
*/
@ -42,7 +42,9 @@ public class DynamoUtils {
throw new IllegalArgumentException("Byte buffer attributeValues cannot be null or empty.");
}
return AttributeValue.builder().b(SdkBytes.fromByteArray(byteBufferValue)).build();
return AttributeValue.builder()
.b(SdkBytes.fromByteArray(byteBufferValue))
.build();
}
public static AttributeValue createAttributeValue(String stringValue) {
@ -97,5 +99,4 @@ public class DynamoUtils {
return av.ss();
}
}
}

397
amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/HierarchicalShardSyncer.java
View file

@ -32,12 +32,11 @@ import com.google.common.annotations.VisibleForTesting;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.Accessors;
import org.apache.commons.lang3.StringUtils;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.experimental.Accessors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import software.amazon.awssdk.services.kinesis.model.ChildShard;
import software.amazon.awssdk.services.kinesis.model.ResourceNotFoundException;
import software.amazon.awssdk.services.kinesis.model.Shard;
@ -78,7 +77,8 @@ public class HierarchicalShardSyncer {
private final DeletedStreamListProvider deletedStreamListProvider;
private static final String MIN_HASH_KEY = BigInteger.ZERO.toString();
private static final String MAX_HASH_KEY = new BigInteger("2").pow(128).subtract(BigInteger.ONE).toString();
private static final String MAX_HASH_KEY =
new BigInteger("2").pow(128).subtract(BigInteger.ONE).toString();
private static final int RETRIES_FOR_COMPLETE_HASH_RANGE = 3;
private static final long DELAY_BETWEEN_LIST_SHARDS_MILLIS = 1000;
@ -91,7 +91,9 @@ public class HierarchicalShardSyncer {
this(isMultiStreamMode, streamIdentifier, null);
}
public HierarchicalShardSyncer(final boolean isMultiStreamMode, final String streamIdentifier,
public HierarchicalShardSyncer(
final boolean isMultiStreamMode,
final String streamIdentifier,
final DeletedStreamListProvider deletedStreamListProvider) {
this.isMultiStreamMode = isMultiStreamMode;
this.streamIdentifier = streamIdentifier;
@ -99,9 +101,7 @@ public class HierarchicalShardSyncer {
}
private static String getShardIdFromLease(Lease lease, MultiStreamArgs multiStreamArgs) {
return multiStreamArgs.isMultiStreamMode()
? ((MultiStreamLease) lease).shardId()
: lease.leaseKey();
return multiStreamArgs.isMultiStreamMode() ? ((MultiStreamLease) lease).shardId() : lease.leaseKey();
}
/**
@ -119,23 +119,41 @@ public class HierarchicalShardSyncer {
* @throws ProvisionedThroughputException
* @throws KinesisClientLibIOException
*/
public synchronized boolean checkAndCreateLeaseForNewShards(@NonNull final ShardDetector shardDetector,
final LeaseRefresher leaseRefresher, final InitialPositionInStreamExtended initialPosition,
final MetricsScope scope, final boolean ignoreUnexpectedChildShards, final boolean isLeaseTableEmpty)
throws DependencyException, InvalidStateException, ProvisionedThroughputException, KinesisClientLibIOException, InterruptedException {
final List<Shard> latestShards = isLeaseTableEmpty ?
getShardListAtInitialPosition(shardDetector, initialPosition) : getShardList(shardDetector);
return checkAndCreateLeaseForNewShards(shardDetector, leaseRefresher, initialPosition, latestShards, ignoreUnexpectedChildShards, scope,
public synchronized boolean checkAndCreateLeaseForNewShards(
@NonNull final ShardDetector shardDetector,
final LeaseRefresher leaseRefresher,
final InitialPositionInStreamExtended initialPosition,
final MetricsScope scope,
final boolean ignoreUnexpectedChildShards,
final boolean isLeaseTableEmpty)
throws DependencyException, InvalidStateException, ProvisionedThroughputException,
KinesisClientLibIOException, InterruptedException {
final List<Shard> latestShards = isLeaseTableEmpty
? getShardListAtInitialPosition(shardDetector, initialPosition)
: getShardList(shardDetector);
return checkAndCreateLeaseForNewShards(
shardDetector,
leaseRefresher,
initialPosition,
latestShards,
ignoreUnexpectedChildShards,
scope,
isLeaseTableEmpty);
}
/**
* Provide a pre-collected list of shards to avoid calling ListShards API
*/
public synchronized boolean checkAndCreateLeaseForNewShards(@NonNull final ShardDetector shardDetector,
final LeaseRefresher leaseRefresher, final InitialPositionInStreamExtended initialPosition,
List<Shard> latestShards, final boolean ignoreUnexpectedChildShards, final MetricsScope scope, final boolean isLeaseTableEmpty)
throws DependencyException, InvalidStateException, ProvisionedThroughputException, KinesisClientLibIOException {
public synchronized boolean checkAndCreateLeaseForNewShards(
@NonNull final ShardDetector shardDetector,
final LeaseRefresher leaseRefresher,
final InitialPositionInStreamExtended initialPosition,
List<Shard> latestShards,
final boolean ignoreUnexpectedChildShards,
final MetricsScope scope,
final boolean isLeaseTableEmpty)
throws DependencyException, InvalidStateException, ProvisionedThroughputException,
KinesisClientLibIOException {
if (!CollectionUtils.isNullOrEmpty(latestShards)) {
log.debug("{} - Num shards: {}", streamIdentifier, latestShards.size());
} else {
@ -144,19 +162,22 @@ public class HierarchicalShardSyncer {
}
final Map<String, Shard> shardIdToShardMap = constructShardIdToShardMap(latestShards);
final Map<String, Set<String>> shardIdToChildShardIdsMap = constructShardIdToChildShardIdsMap(
shardIdToShardMap);
final Map<String, Set<String>> shardIdToChildShardIdsMap =
constructShardIdToChildShardIdsMap(shardIdToShardMap);
final Set<String> inconsistentShardIds = findInconsistentShardIds(shardIdToChildShardIdsMap, shardIdToShardMap);
if (!ignoreUnexpectedChildShards) {
assertAllParentShardsAreClosed(inconsistentShardIds);
}
final List<Lease> currentLeases = isMultiStreamMode ?
leaseRefresher.listLeasesForStream(shardDetector.streamIdentifier()) : leaseRefresher.listLeases();
final MultiStreamArgs multiStreamArgs = new MultiStreamArgs(isMultiStreamMode, shardDetector.streamIdentifier());
final LeaseSynchronizer leaseSynchronizer = isLeaseTableEmpty ? new EmptyLeaseTableSynchronizer() :
new NonEmptyLeaseTableSynchronizer(shardDetector, shardIdToShardMap, shardIdToChildShardIdsMap);
final List<Lease> newLeasesToCreate = determineNewLeasesToCreate(leaseSynchronizer, latestShards, currentLeases,
initialPosition, inconsistentShardIds, multiStreamArgs);
final List<Lease> currentLeases = isMultiStreamMode
? leaseRefresher.listLeasesForStream(shardDetector.streamIdentifier())
: leaseRefresher.listLeases();
final MultiStreamArgs multiStreamArgs =
new MultiStreamArgs(isMultiStreamMode, shardDetector.streamIdentifier());
final LeaseSynchronizer leaseSynchronizer = isLeaseTableEmpty
? new EmptyLeaseTableSynchronizer()
: new NonEmptyLeaseTableSynchronizer(shardDetector, shardIdToShardMap, shardIdToChildShardIdsMap);
final List<Lease> newLeasesToCreate = determineNewLeasesToCreate(
leaseSynchronizer, latestShards, currentLeases, initialPosition, inconsistentShardIds, multiStreamArgs);
log.info("{} - Number of new leases to create: {}", streamIdentifier, newLeasesToCreate.size());
final Set<Lease> createdLeases = new HashSet<>();
@ -172,7 +193,9 @@ public class HierarchicalShardSyncer {
} finally {
MetricsUtil.addSuccessAndLatency(scope, "CreateLease", success, startTime, MetricsLevel.DETAILED);
if (lease.checkpoint() != null) {
final String metricName = lease.checkpoint().isSentinelCheckpoint() ? lease.checkpoint().sequenceNumber() : "SEQUENCE_NUMBER";
final String metricName = lease.checkpoint().isSentinelCheckpoint()
? lease.checkpoint().sequenceNumber()
: "SEQUENCE_NUMBER";
MetricsUtil.addSuccess(scope, "CreateLease_" + metricName, true, MetricsLevel.DETAILED);
}
}
@ -187,7 +210,7 @@ public class HierarchicalShardSyncer {
* @throws KinesisClientLibIOException
*/
private static void assertAllParentShardsAreClosed(final Set<String> inconsistentShardIds)
throws KinesisClientLibIOException {
throws KinesisClientLibIOException {
if (!CollectionUtils.isNullOrEmpty(inconsistentShardIds)) {
final String ids = StringUtils.join(inconsistentShardIds, ' ');
throw new KinesisClientLibIOException(String.format(
@ -205,12 +228,17 @@ public class HierarchicalShardSyncer {
* @param shardIdToShardMap
* @return Set of inconsistent open shard ids for shards having open parents.
*/
private static Set<String> findInconsistentShardIds(final Map<String, Set<String>> shardIdToChildShardIdsMap,
final Map<String, Shard> shardIdToShardMap) {
private static Set<String> findInconsistentShardIds(
final Map<String, Set<String>> shardIdToChildShardIdsMap, final Map<String, Shard> shardIdToShardMap) {
return shardIdToChildShardIdsMap.entrySet().stream()
.filter(entry -> entry.getKey() == null
|| shardIdToShardMap.get(entry.getKey()).sequenceNumberRange().endingSequenceNumber() == null)
.flatMap(entry -> shardIdToChildShardIdsMap.get(entry.getKey()).stream()).collect(Collectors.toSet());
|| shardIdToShardMap
.get(entry.getKey())
.sequenceNumberRange()
.endingSequenceNumber()
== null)
.flatMap(entry -> shardIdToChildShardIdsMap.get(entry.getKey()).stream())
.collect(Collectors.toSet());
}
/**
@ -227,15 +255,15 @@ public class HierarchicalShardSyncer {
final Shard shard = entry.getValue();
final String parentShardId = shard.parentShardId();
if (parentShardId != null && shardIdToShardMap.containsKey(parentShardId)) {
final Set<String> childShardIds = shardIdToChildShardIdsMap.computeIfAbsent(parentShardId,
key -> new HashSet<>());
final Set<String> childShardIds =
shardIdToChildShardIdsMap.computeIfAbsent(parentShardId, key -> new HashSet<>());
childShardIds.add(shardId);
}
final String adjacentParentShardId = shard.adjacentParentShardId();
if (adjacentParentShardId != null && shardIdToShardMap.containsKey(adjacentParentShardId)) {
final Set<String> childShardIds = shardIdToChildShardIdsMap.computeIfAbsent(adjacentParentShardId,
key -> new HashSet<>());
final Set<String> childShardIds =
shardIdToChildShardIdsMap.computeIfAbsent(adjacentParentShardId, key -> new HashSet<>());
childShardIds.add(shardId);
}
}
@ -247,7 +275,8 @@ public class HierarchicalShardSyncer {
* @param initialPositionInStreamExtended
* @return ShardFilter shard filter for the corresponding position in the stream.
*/
private static ShardFilter getShardFilterFromInitialPosition(InitialPositionInStreamExtended initialPositionInStreamExtended) {
private static ShardFilter getShardFilterFromInitialPosition(
InitialPositionInStreamExtended initialPositionInStreamExtended) {
ShardFilter.Builder builder = ShardFilter.builder();
switch (initialPositionInStreamExtended.getInitialPositionInStream()) {
@ -258,14 +287,17 @@ public class HierarchicalShardSyncer {
builder = builder.type(ShardFilterType.AT_TRIM_HORIZON);
break;
case AT_TIMESTAMP:
builder = builder.type(ShardFilterType.AT_TIMESTAMP).timestamp(initialPositionInStreamExtended.getTimestamp().toInstant());
builder = builder.type(ShardFilterType.AT_TIMESTAMP)
.timestamp(
initialPositionInStreamExtended.getTimestamp().toInstant());
break;
}
return builder.build();
}
private static List<Shard> getShardListAtInitialPosition(@NonNull final ShardDetector shardDetector,
InitialPositionInStreamExtended initialPositionInStreamExtended) throws KinesisClientLibIOException, InterruptedException {
private static List<Shard> getShardListAtInitialPosition(
@NonNull final ShardDetector shardDetector, InitialPositionInStreamExtended initialPositionInStreamExtended)
throws KinesisClientLibIOException, InterruptedException {
final ShardFilter shardFilter = getShardFilterFromInitialPosition(initialPositionInStreamExtended);
final String streamName = shardDetector.streamIdentifier().streamName();
@ -276,8 +308,8 @@ public class HierarchicalShardSyncer {
shards = shardDetector.listShardsWithFilter(shardFilter);
if (shards == null) {
throw new KinesisClientLibIOException(
"Stream " + streamName + " is not in ACTIVE OR UPDATING state - will retry getting the shard list.");
throw new KinesisClientLibIOException("Stream " + streamName
+ " is not in ACTIVE OR UPDATING state - will retry getting the shard list.");
}
if (isHashRangeOfShardsComplete(shards)) {
@ -287,13 +319,13 @@ public class HierarchicalShardSyncer {
Thread.sleep(DELAY_BETWEEN_LIST_SHARDS_MILLIS);
}
throw new KinesisClientLibIOException("Hash range of shards returned for " + streamName + " was incomplete after "
+ RETRIES_FOR_COMPLETE_HASH_RANGE + " retries.");
throw new KinesisClientLibIOException("Hash range of shards returned for " + streamName
+ " was incomplete after " + RETRIES_FOR_COMPLETE_HASH_RANGE + " retries.");
}
private List<Shard> getShardList(@NonNull final ShardDetector shardDetector) throws KinesisClientLibIOException {
// Fallback to existing behavior for backward compatibility
List<Shard> shardList = Collections.emptyList();
List<Shard> shardList = Collections.emptyList();
try {
shardList = shardDetector.listShardsWithoutConsumingResourceNotFoundException();
} catch (ResourceNotFoundException e) {
@ -303,8 +335,9 @@ public class HierarchicalShardSyncer {
}
final Optional<List<Shard>> shards = Optional.of(shardList);
return shards.orElseThrow(() -> new KinesisClientLibIOException("Stream " + shardDetector.streamIdentifier().streamName() +
" is not in ACTIVE OR UPDATING state - will retry getting the shard list."));
return shards.orElseThrow(() -> new KinesisClientLibIOException(
"Stream " + shardDetector.streamIdentifier().streamName()
+ " is not in ACTIVE OR UPDATING state - will retry getting the shard list."));
}
private static boolean isHashRangeOfShardsComplete(@NonNull List<Shard> shards) {
@ -315,8 +348,8 @@ public class HierarchicalShardSyncer {
final Comparator<Shard> shardStartingHashKeyBasedComparator = new ShardStartingHashKeyBasedComparator();
shards.sort(shardStartingHashKeyBasedComparator);
if (!shards.get(0).hashKeyRange().startingHashKey().equals(MIN_HASH_KEY) ||
!shards.get(shards.size() - 1).hashKeyRange().endingHashKey().equals(MAX_HASH_KEY)) {
if (!shards.get(0).hashKeyRange().startingHashKey().equals(MIN_HASH_KEY)
|| !shards.get(shards.size() - 1).hashKeyRange().endingHashKey().equals(MAX_HASH_KEY)) {
return false;
}
@ -324,11 +357,16 @@ public class HierarchicalShardSyncer {
for (int i = 1; i < shards.size(); i++) {
final Shard shardAtStartOfPossibleHole = shards.get(i - 1);
final Shard shardAtEndOfPossibleHole = shards.get(i);
final BigInteger startOfPossibleHole = new BigInteger(shardAtStartOfPossibleHole.hashKeyRange().endingHashKey());
final BigInteger endOfPossibleHole = new BigInteger(shardAtEndOfPossibleHole.hashKeyRange().startingHashKey());
final BigInteger startOfPossibleHole =
new BigInteger(shardAtStartOfPossibleHole.hashKeyRange().endingHashKey());
final BigInteger endOfPossibleHole =
new BigInteger(shardAtEndOfPossibleHole.hashKeyRange().startingHashKey());
if (!endOfPossibleHole.subtract(startOfPossibleHole).equals(BigInteger.ONE)) {
log.error("Incomplete hash range found between {} and {}.", shardAtStartOfPossibleHole, shardAtEndOfPossibleHole);
log.error(
"Incomplete hash range found between {} and {}.",
shardAtStartOfPossibleHole,
shardAtEndOfPossibleHole);
return false;
}
}
@ -350,10 +388,15 @@ public class HierarchicalShardSyncer {
* @param multiStreamArgs determines if we are using multistream mode.
* @return List of new leases to create sorted by starting sequenceNumber of the corresponding shard
*/
static List<Lease> determineNewLeasesToCreate(final LeaseSynchronizer leaseSynchronizer, final List<Shard> shards,
final List<Lease> currentLeases, final InitialPositionInStreamExtended initialPosition,
final Set<String> inconsistentShardIds, final MultiStreamArgs multiStreamArgs) {
return leaseSynchronizer.determineNewLeasesToCreate(shards, currentLeases, initialPosition, inconsistentShardIds, multiStreamArgs);
static List<Lease> determineNewLeasesToCreate(
final LeaseSynchronizer leaseSynchronizer,
final List<Shard> shards,
final List<Lease> currentLeases,
final InitialPositionInStreamExtended initialPosition,
final Set<String> inconsistentShardIds,
final MultiStreamArgs multiStreamArgs) {
return leaseSynchronizer.determineNewLeasesToCreate(
shards, currentLeases, initialPosition, inconsistentShardIds, multiStreamArgs);
}
/**
@ -368,10 +411,18 @@ public class HierarchicalShardSyncer {
* @param inconsistentShardIds Set of child shard ids having open parents.
* @return List of new leases to create sorted by starting sequenceNumber of the corresponding shard
*/
static List<Lease> determineNewLeasesToCreate(final LeaseSynchronizer leaseSynchronizer, final List<Shard> shards,
final List<Lease> currentLeases, final InitialPositionInStreamExtended initialPosition,
static List<Lease> determineNewLeasesToCreate(
final LeaseSynchronizer leaseSynchronizer,
final List<Shard> shards,
final List<Lease> currentLeases,
final InitialPositionInStreamExtended initialPosition,
final Set<String> inconsistentShardIds) {
return determineNewLeasesToCreate(leaseSynchronizer, shards, currentLeases, initialPosition, inconsistentShardIds,
return determineNewLeasesToCreate(
leaseSynchronizer,
shards,
currentLeases,
initialPosition,
inconsistentShardIds,
new MultiStreamArgs(false, null));
}
@ -386,10 +437,14 @@ public class HierarchicalShardSyncer {
* location in the shard (when an application starts up for the first time - and there are no checkpoints).
* @return List of new leases to create sorted by starting sequenceNumber of the corresponding shard
*/
static List<Lease> determineNewLeasesToCreate(final LeaseSynchronizer leaseSynchronizer, final List<Shard> shards,
final List<Lease> currentLeases, final InitialPositionInStreamExtended initialPosition) {
static List<Lease> determineNewLeasesToCreate(
final LeaseSynchronizer leaseSynchronizer,
final List<Shard> shards,
final List<Lease> currentLeases,
final InitialPositionInStreamExtended initialPosition) {
final Set<String> inconsistentShardIds = new HashSet<>();
return determineNewLeasesToCreate(leaseSynchronizer, shards, currentLeases, initialPosition, inconsistentShardIds);
return determineNewLeasesToCreate(
leaseSynchronizer, shards, currentLeases, initialPosition, inconsistentShardIds);
}
/**
@ -407,10 +462,13 @@ public class HierarchicalShardSyncer {
* @param memoizationContext Memoization of shards that have been evaluated as part of the evaluation
* @return true if the shard is a descendant of any current shard (lease already exists)
*/
static boolean checkIfDescendantAndAddNewLeasesForAncestors(final String shardId,
final InitialPositionInStreamExtended initialPosition, final Set<String> shardIdsOfCurrentLeases,
static boolean checkIfDescendantAndAddNewLeasesForAncestors(
final String shardId,
final InitialPositionInStreamExtended initialPosition,
final Set<String> shardIdsOfCurrentLeases,
final Map<String, Shard> shardIdToShardMapOfAllKinesisShards,
final Map<String, Lease> shardIdToLeaseMapOfNewShards, final MemoizationContext memoizationContext,
final Map<String, Lease> shardIdToLeaseMapOfNewShards,
final MemoizationContext memoizationContext,
final MultiStreamArgs multiStreamArgs) {
final String streamIdentifier = getStreamIdentifier(multiStreamArgs);
final Boolean previousValue = memoizationContext.isDescendant(shardId);
@ -427,7 +485,10 @@ public class HierarchicalShardSyncer {
isDescendant = true;
// We don't need to add leases of its ancestors,
// because we'd have done it when creating a lease for this shard.
log.debug("{} - Shard {} is a descendant shard of an existing shard. Skipping lease creation", streamIdentifier, shardId);
log.debug(
"{} - Shard {} is a descendant shard of an existing shard. Skipping lease creation",
streamIdentifier,
shardId);
} else {
final Shard shard = shardIdToShardMapOfAllKinesisShards.get(shardId);
@ -436,9 +497,14 @@ public class HierarchicalShardSyncer {
// Check if the parent is a descendant, and include its ancestors. Or, if the parent is NOT a
// descendant but we should create a lease for it anyway (e.g. to include in processing from
// TRIM_HORIZON or AT_TIMESTAMP). If either is true, then we mark the current shard as a descendant.
final boolean isParentDescendant = checkIfDescendantAndAddNewLeasesForAncestors(parentShardId,
initialPosition, shardIdsOfCurrentLeases, shardIdToShardMapOfAllKinesisShards,
shardIdToLeaseMapOfNewShards, memoizationContext, multiStreamArgs);
final boolean isParentDescendant = checkIfDescendantAndAddNewLeasesForAncestors(
parentShardId,
initialPosition,
shardIdsOfCurrentLeases,
shardIdToShardMapOfAllKinesisShards,
shardIdToLeaseMapOfNewShards,
memoizationContext,
multiStreamArgs);
if (isParentDescendant || memoizationContext.shouldCreateLease(parentShardId)) {
isDescendant = true;
descendantParentShardIds.add(parentShardId);
@ -465,13 +531,17 @@ public class HierarchicalShardSyncer {
* therefore covered in the lease table). So we should create a lease for the parent.
*/
if (lease == null) {
if (memoizationContext.shouldCreateLease(parentShardId) ||
!descendantParentShardIds.contains(parentShardId)) {
log.debug("{} : Need to create a lease for shardId {}", streamIdentifier, parentShardId);
lease = multiStreamArgs.isMultiStreamMode() ?
newKCLMultiStreamLease(shardIdToShardMapOfAllKinesisShards.get(parentShardId),
multiStreamArgs.streamIdentifier()) :
newKCLLease(shardIdToShardMapOfAllKinesisShards.get(parentShardId));
if (memoizationContext.shouldCreateLease(parentShardId)
|| !descendantParentShardIds.contains(parentShardId)) {
log.debug(
"{} : Need to create a lease for shardId {}",
streamIdentifier,
parentShardId);
lease = multiStreamArgs.isMultiStreamMode()
? newKCLMultiStreamLease(
shardIdToShardMapOfAllKinesisShards.get(parentShardId),
multiStreamArgs.streamIdentifier())
: newKCLLease(shardIdToShardMapOfAllKinesisShards.get(parentShardId));
shardIdToLeaseMapOfNewShards.put(parentShardId, lease);
}
}
@ -502,15 +572,21 @@ public class HierarchicalShardSyncer {
*/
if (lease != null) {
if (descendantParentShardIds.contains(parentShardId)
&& !initialPosition.getInitialPositionInStream()
.equals(InitialPositionInStream.AT_TIMESTAMP)) {
log.info("Setting Lease '{}' checkpoint to 'TRIM_HORIZON'. Checkpoint was previously set to {}",
lease.leaseKey(), lease.checkpoint());
&& !initialPosition
.getInitialPositionInStream()
.equals(InitialPositionInStream.AT_TIMESTAMP)) {
log.info(
"Setting Lease '{}' checkpoint to 'TRIM_HORIZON'. Checkpoint was previously set to {}",
lease.leaseKey(),
lease.checkpoint());
lease.checkpoint(ExtendedSequenceNumber.TRIM_HORIZON);
} else {
final ExtendedSequenceNumber newCheckpoint = convertToCheckpoint(initialPosition);
log.info("Setting Lease '{}' checkpoint to '{}'. Checkpoint was previously set to {}",
lease.leaseKey(), newCheckpoint, lease.checkpoint());
log.info(
"Setting Lease '{}' checkpoint to '{}'. Checkpoint was previously set to {}",
lease.leaseKey(),
newCheckpoint,
lease.checkpoint());
lease.checkpoint(newCheckpoint);
}
}
@ -522,8 +598,9 @@ public class HierarchicalShardSyncer {
// lease just like we do for TRIM_HORIZON. However we will only return back records with server-side
// timestamp at or after the specified initial position timestamp.
if (initialPosition.getInitialPositionInStream().equals(InitialPositionInStream.TRIM_HORIZON)
|| initialPosition.getInitialPositionInStream()
.equals(InitialPositionInStream.AT_TIMESTAMP)) {
|| initialPosition
.getInitialPositionInStream()
.equals(InitialPositionInStream.AT_TIMESTAMP)) {
memoizationContext.setShouldCreateLease(shardId, true);
}
}
@ -534,12 +611,20 @@ public class HierarchicalShardSyncer {
return isDescendant;
}
static boolean checkIfDescendantAndAddNewLeasesForAncestors(final String shardId,
final InitialPositionInStreamExtended initialPosition, final Set<String> shardIdsOfCurrentLeases,
static boolean checkIfDescendantAndAddNewLeasesForAncestors(
final String shardId,
final InitialPositionInStreamExtended initialPosition,
final Set<String> shardIdsOfCurrentLeases,
final Map<String, Shard> shardIdToShardMapOfAllKinesisShards,
final Map<String, Lease> shardIdToLeaseMapOfNewShards, MemoizationContext memoizationContext) {
return checkIfDescendantAndAddNewLeasesForAncestors(shardId, initialPosition, shardIdsOfCurrentLeases,
shardIdToShardMapOfAllKinesisShards, shardIdToLeaseMapOfNewShards, memoizationContext,
final Map<String, Lease> shardIdToLeaseMapOfNewShards,
MemoizationContext memoizationContext) {
return checkIfDescendantAndAddNewLeasesForAncestors(
shardId,
initialPosition,
shardIdsOfCurrentLeases,
shardIdToShardMapOfAllKinesisShards,
shardIdToLeaseMapOfNewShards,
memoizationContext,
new MultiStreamArgs(false, null));
}
@ -552,8 +637,8 @@ public class HierarchicalShardSyncer {
* @param shardIdToShardMapOfAllKinesisShards ShardId->Shard map containing all shards obtained via DescribeStream.
* @return Set of parentShardIds
*/
static Set<String> getParentShardIds(final Shard shard,
final Map<String, Shard> shardIdToShardMapOfAllKinesisShards) {
static Set<String> getParentShardIds(
final Shard shard, final Map<String, Shard> shardIdToShardMapOfAllKinesisShards) {
final Set<String> parentShardIds = new HashSet<>(2);
final String parentShardId = shard.parentShardId();
if (parentShardId != null && shardIdToShardMapOfAllKinesisShards.containsKey(parentShardId)) {
@ -566,12 +651,13 @@ public class HierarchicalShardSyncer {
return parentShardIds;
}
public synchronized Lease createLeaseForChildShard(final ChildShard childShard,
final StreamIdentifier streamIdentifier) throws InvalidStateException {
public synchronized Lease createLeaseForChildShard(
final ChildShard childShard, final StreamIdentifier streamIdentifier) throws InvalidStateException {
final MultiStreamArgs multiStreamArgs = new MultiStreamArgs(isMultiStreamMode, streamIdentifier);
return multiStreamArgs.isMultiStreamMode() ? newKCLMultiStreamLeaseForChildShard(childShard, streamIdentifier)
: newKCLLeaseForChildShard(childShard);
return multiStreamArgs.isMultiStreamMode()
? newKCLMultiStreamLeaseForChildShard(childShard, streamIdentifier)
: newKCLLeaseForChildShard(childShard);
}
/**
@ -595,8 +681,8 @@ public class HierarchicalShardSyncer {
return newLease;
}
private static Lease newKCLMultiStreamLeaseForChildShard(final ChildShard childShard,
final StreamIdentifier streamIdentifier) throws InvalidStateException {
private static Lease newKCLMultiStreamLeaseForChildShard(
final ChildShard childShard, final StreamIdentifier streamIdentifier) throws InvalidStateException {
MultiStreamLease newLease = new MultiStreamLease();
newLease.leaseKey(MultiStreamLease.getLeaseKey(streamIdentifier.serialize(), childShard.shardId()));
if (!CollectionUtils.isNullOrEmpty(childShard.parentShards())) {
@ -671,8 +757,10 @@ public class HierarchicalShardSyncer {
* @return List of open shards (shards at the tip of the stream) - may include shards that are not yet active.
*/
static List<Shard> getOpenShards(final List<Shard> allShards, final String streamIdentifier) {
return allShards.stream().filter(shard -> shard.sequenceNumberRange().endingSequenceNumber() == null)
.peek(shard -> log.debug("{} : Found open shard: {}", streamIdentifier, shard.shardId())).collect(Collectors.toList());
return allShards.stream()
.filter(shard -> shard.sequenceNumberRange().endingSequenceNumber() == null)
.peek(shard -> log.debug("{} : Found open shard: {}", streamIdentifier, shard.shardId()))
.collect(Collectors.toList());
}
private static ExtendedSequenceNumber convertToCheckpoint(final InitialPositionInStreamExtended position) {
@ -691,7 +779,8 @@ public class HierarchicalShardSyncer {
private static String getStreamIdentifier(MultiStreamArgs multiStreamArgs) {
return Optional.ofNullable(multiStreamArgs.streamIdentifier())
.map(streamId -> streamId.serialize()).orElse("single_stream_mode");
.map(streamId -> streamId.serialize())
.orElse("single_stream_mode");
}
/**
@ -745,8 +834,10 @@ public class HierarchicalShardSyncer {
// If we found shards for the two leases, use comparison of the starting sequence numbers
if (shard1 != null && shard2 != null) {
BigInteger sequenceNumber1 = new BigInteger(shard1.sequenceNumberRange().startingSequenceNumber());
BigInteger sequenceNumber2 = new BigInteger(shard2.sequenceNumberRange().startingSequenceNumber());
BigInteger sequenceNumber1 =
new BigInteger(shard1.sequenceNumberRange().startingSequenceNumber());
BigInteger sequenceNumber2 =
new BigInteger(shard2.sequenceNumberRange().startingSequenceNumber());
result = sequenceNumber1.compareTo(sequenceNumber2);
}
@ -780,9 +871,12 @@ public class HierarchicalShardSyncer {
* @param multiStreamArgs
* @return
*/
List<Lease> determineNewLeasesToCreate(List<Shard> shards, List<Lease> currentLeases,
InitialPositionInStreamExtended initialPosition, Set<String> inconsistentShardIds,
MultiStreamArgs multiStreamArgs);
List<Lease> determineNewLeasesToCreate(
List<Shard> shards,
List<Lease> currentLeases,
InitialPositionInStreamExtended initialPosition,
Set<String> inconsistentShardIds,
MultiStreamArgs multiStreamArgs);
}
/**
@ -805,21 +899,29 @@ public class HierarchicalShardSyncer {
* @return
*/
@Override
public List<Lease> determineNewLeasesToCreate(List<Shard> shards, List<Lease> currentLeases,
InitialPositionInStreamExtended initialPosition, Set<String> inconsistentShardIds, MultiStreamArgs multiStreamArgs) {
public List<Lease> determineNewLeasesToCreate(
List<Shard> shards,
List<Lease> currentLeases,
InitialPositionInStreamExtended initialPosition,
Set<String> inconsistentShardIds,
MultiStreamArgs multiStreamArgs) {
final String streamIdentifier = Optional.ofNullable(multiStreamArgs.streamIdentifier())
.map(streamId -> streamId.serialize()).orElse("");
.map(streamId -> streamId.serialize())
.orElse("");
final Map<String, Shard> shardIdToShardMapOfAllKinesisShards = constructShardIdToShardMap(shards);
currentLeases.stream().peek(lease -> log.debug("{} : Existing lease: {}", streamIdentifier, lease))
currentLeases.stream()
.peek(lease -> log.debug("{} : Existing lease: {}", streamIdentifier, lease))
.map(lease -> getShardIdFromLease(lease, multiStreamArgs))
.collect(Collectors.toSet());
final List<Lease> newLeasesToCreate = getLeasesToCreateForOpenAndClosedShards(initialPosition, shards, multiStreamArgs, streamIdentifier);
final List<Lease> newLeasesToCreate =
getLeasesToCreateForOpenAndClosedShards(initialPosition, shards, multiStreamArgs, streamIdentifier);
//TODO: Verify before LTR launch that ending sequence number is still returned from the service.
// TODO: Verify before LTR launch that ending sequence number is still returned from the service.
final Comparator<Lease> startingSequenceNumberComparator =
new StartingSequenceNumberAndShardIdBasedComparator(shardIdToShardMapOfAllKinesisShards, multiStreamArgs);
new StartingSequenceNumberAndShardIdBasedComparator(
shardIdToShardMapOfAllKinesisShards, multiStreamArgs);
newLeasesToCreate.sort(startingSequenceNumberComparator);
return newLeasesToCreate;
}
@ -829,14 +931,18 @@ public class HierarchicalShardSyncer {
* regardless of if they are open or closed. Closed shards will be unblocked via child shard information upon
* reaching SHARD_END.
*/
private List<Lease> getLeasesToCreateForOpenAndClosedShards(InitialPositionInStreamExtended initialPosition,
List<Shard> shards, MultiStreamArgs multiStreamArgs, String streamId) {
private List<Lease> getLeasesToCreateForOpenAndClosedShards(
InitialPositionInStreamExtended initialPosition,
List<Shard> shards,
MultiStreamArgs multiStreamArgs,
String streamId) {
final Map<String, Lease> shardIdToNewLeaseMap = new HashMap<>();
for (Shard shard : shards) {
final String shardId = shard.shardId();
final Lease lease = multiStreamArgs.isMultiStreamMode() ?
newKCLMultiStreamLease(shard, multiStreamArgs.streamIdentifier) : newKCLLease(shard);
final Lease lease = multiStreamArgs.isMultiStreamMode()
? newKCLMultiStreamLease(shard, multiStreamArgs.streamIdentifier)
: newKCLLease(shard);
lease.checkpoint(convertToCheckpoint(initialPosition));
log.debug("{} : Need to create a lease for shard with shardId {}", streamId, shardId);
@ -909,13 +1015,17 @@ public class HierarchicalShardSyncer {
* @return List of new leases to create sorted by starting sequenceNumber of the corresponding shard
*/
@Override
public synchronized List<Lease> determineNewLeasesToCreate(final List<Shard> shards, final List<Lease> currentLeases,
final InitialPositionInStreamExtended initialPosition, final Set<String> inconsistentShardIds,
public synchronized List<Lease> determineNewLeasesToCreate(
final List<Shard> shards,
final List<Lease> currentLeases,
final InitialPositionInStreamExtended initialPosition,
final Set<String> inconsistentShardIds,
final MultiStreamArgs multiStreamArgs) {
final Map<String, Lease> shardIdToNewLeaseMap = new HashMap<>();
final Map<String, Shard> shardIdToShardMapOfAllKinesisShards = constructShardIdToShardMap(shards);
final String streamIdentifier = Optional.ofNullable(multiStreamArgs.streamIdentifier())
.map(streamId -> streamId.serialize()).orElse("");
.map(streamId -> streamId.serialize())
.orElse("");
final Set<String> shardIdsOfCurrentLeases = currentLeases.stream()
.peek(lease -> log.debug("{} : Existing lease: {}", streamIdentifier, lease))
.map(lease -> getShardIdFromLease(lease, multiStreamArgs))
@ -929,9 +1039,15 @@ public class HierarchicalShardSyncer {
final String shardId = shard.shardId();
log.debug("{} : Evaluating leases for open shard {} and its ancestors.", streamIdentifier, shardId);
if (shardIdsOfCurrentLeases.contains(shardId)) {
log.debug("{} : Lease for shardId {} already exists. Not creating a lease", streamIdentifier, shardId);
log.debug(
"{} : Lease for shardId {} already exists. Not creating a lease",
streamIdentifier,
shardId);
} else if (inconsistentShardIds.contains(shardId)) {
log.info("{} : shardId {} is an inconsistent child. Not creating a lease", streamIdentifier, shardId);
log.info(
"{} : shardId {} is an inconsistent child. Not creating a lease",
streamIdentifier,
shardId);
} else {
log.debug("{} : Beginning traversal of ancestry tree for shardId {}", streamIdentifier, shardId);
@ -939,9 +1055,14 @@ public class HierarchicalShardSyncer {
// We will create leases for only one level in the ancestry tree. Once we find the first ancestor
// that needs to be processed in order to complete the hash range, we will not create leases for
// further descendants of that ancestor.
final boolean isDescendant = checkIfDescendantAndAddNewLeasesForAncestors(shardId, initialPosition,
shardIdsOfCurrentLeases, shardIdToShardMapOfAllKinesisShards, shardIdToNewLeaseMap,
memoizationContext, multiStreamArgs);
final boolean isDescendant = checkIfDescendantAndAddNewLeasesForAncestors(
shardId,
initialPosition,
shardIdsOfCurrentLeases,
shardIdToShardMapOfAllKinesisShards,
shardIdToNewLeaseMap,
memoizationContext,
multiStreamArgs);
// If shard is a descendant, the leases for its ancestors were already created above. Open shards
// that are NOT descendants will not have leases yet, so we create them here. We will not create
@ -949,22 +1070,30 @@ public class HierarchicalShardSyncer {
// SHARD_END of their parents.
if (!isDescendant) {
log.debug("{} : shardId {} has no ancestors. Creating a lease.", streamIdentifier, shardId);
final Lease newLease = multiStreamArgs.isMultiStreamMode() ?
newKCLMultiStreamLease(shard, multiStreamArgs.streamIdentifier()) :
newKCLLease(shard);
final Lease newLease = multiStreamArgs.isMultiStreamMode()
? newKCLMultiStreamLease(shard, multiStreamArgs.streamIdentifier())
: newKCLLease(shard);
newLease.checkpoint(convertToCheckpoint(initialPosition));
log.debug("{} : Set checkpoint of {} to {}", streamIdentifier, newLease.leaseKey(), newLease.checkpoint());
log.debug(
"{} : Set checkpoint of {} to {}",
streamIdentifier,
newLease.leaseKey(),
newLease.checkpoint());
shardIdToNewLeaseMap.put(shardId, newLease);
} else {
log.debug("{} : shardId {} is a descendant whose ancestors should already have leases. " +
"Not creating a lease.", streamIdentifier, shardId);
log.debug(
"{} : shardId {} is a descendant whose ancestors should already have leases. "
+ "Not creating a lease.",
streamIdentifier,
shardId);
}
}
}
final List<Lease> newLeasesToCreate = new ArrayList<>(shardIdToNewLeaseMap.values());
final Comparator<Lease> startingSequenceNumberComparator = new StartingSequenceNumberAndShardIdBasedComparator(
shardIdToShardMapOfAllKinesisShards, multiStreamArgs);
final Comparator<Lease> startingSequenceNumberComparator =
new StartingSequenceNumberAndShardIdBasedComparator(
shardIdToShardMapOfAllKinesisShards, multiStreamArgs);
newLeasesToCreate.sort(startingSequenceNumberComparator);
return newLeasesToCreate;
}

Some files were not shown because too many files have changed in this diff Show more