luciano/amazon-kinesis-client
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(src): delombok the src

Browse source
This commit is contained in:
glarwood 2018-12-13 11:57:02 -08:00
parent 3a91015bba
commit c1b1f9b74d
23 changed files with 452 additions and 166 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
pom.xml
View file

@ -67,12 +67,6 @@
<artifactId>commons-logging</artifactId>
<version>1.1.3</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.16.10</version>
<scope>provided</scope>
</dependency>
<!-- Test -->
<dependency>

35
src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/Checkpoint.java
View file

@ -1,12 +1,13 @@
package com.amazonaws.services.kinesis.clientlibrary.lib.checkpoint;
import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber;
import lombok.Data;
import java.util.Objects;
/**
* A class encapsulating the 2 pieces of state stored in a checkpoint.
*/
@Data public class Checkpoint {
public class Checkpoint {
private final ExtendedSequenceNumber checkpoint;
private final ExtendedSequenceNumber pendingCheckpoint;
@ -24,4 +25,34 @@ import lombok.Data;
this.checkpoint = checkpoint;
this.pendingCheckpoint = pendingCheckpoint;
}
@Override
public String toString() {
return "Checkpoint{" +
"checkpoint=" + checkpoint +
", pendingCheckpoint=" + pendingCheckpoint +
'}';
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Checkpoint that = (Checkpoint) o;
return Objects.equals(checkpoint, that.checkpoint) &&
Objects.equals(pendingCheckpoint, that.pendingCheckpoint);
}
@Override
public int hashCode() {
return Objects.hash(checkpoint, pendingCheckpoint);
}
public ExtendedSequenceNumber getPendingCheckpoint() {
return pendingCheckpoint;
}
public ExtendedSequenceNumber getCheckpoint() {
return checkpoint;
}
}

10
src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/AsynchronousGetRecordsRetrievalStrategy.java
View file

@ -15,6 +15,7 @@
package com.amazonaws.services.kinesis.clientlibrary.lib.worker;
import java.util.HashSet;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
@ -33,15 +34,13 @@ import com.amazonaws.services.kinesis.metrics.impl.ThreadSafeMetricsDelegatingSc
import com.amazonaws.services.kinesis.model.ExpiredIteratorException;
import com.amazonaws.services.kinesis.model.GetRecordsResult;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import lombok.NonNull;
import lombok.extern.apachecommons.CommonsLog;
import org.apache.commons.logging.impl.SimpleLog;
/**
*
*/
@CommonsLog
public class AsynchronousGetRecordsRetrievalStrategy implements GetRecordsRetrievalStrategy {
private static final org.apache.commons.logging.Log log = org.apache.commons.logging.LogFactory.getLog(SimpleLog.class);
private static final int TIME_TO_KEEP_ALIVE = 5;
private static final int CORE_THREAD_POOL_COUNT = 1;
@ -51,9 +50,10 @@ public class AsynchronousGetRecordsRetrievalStrategy implements GetRecordsRetrie
private final String shardId;
final Supplier<CompletionService<DataFetcherResult>> completionServiceSupplier;
public AsynchronousGetRecordsRetrievalStrategy(@NonNull final KinesisDataFetcher dataFetcher,
public AsynchronousGetRecordsRetrievalStrategy(final KinesisDataFetcher dataFetcher,
final int retryGetRecordsInSeconds, final int maxGetRecordsThreadPool, String shardId) {
this(dataFetcher, buildExector(maxGetRecordsThreadPool, shardId), retryGetRecordsInSeconds, shardId);
Objects.requireNonNull(dataFetcher);
}
public AsynchronousGetRecordsRetrievalStrategy(final KinesisDataFetcher dataFetcher,

9
src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/BlockingGetRecordsCache.java
View file

@ -15,20 +15,17 @@
package com.amazonaws.services.kinesis.clientlibrary.lib.worker;
import java.time.Duration;
import java.time.Instant;
import com.amazonaws.services.kinesis.clientlibrary.types.ProcessRecordsInput;
import com.amazonaws.services.kinesis.model.GetRecordsResult;
import lombok.extern.apachecommons.CommonsLog;
import org.apache.commons.logging.impl.SimpleLog;
/**
* This is the BlockingGetRecordsCache class. This class blocks any calls to the getRecords on the
* GetRecordsRetrievalStrategy class.
*/
@CommonsLog
public class BlockingGetRecordsCache implements GetRecordsCache {
private static final org.apache.commons.logging.Log log = org.apache.commons.logging.LogFactory.getLog(SimpleLog.class);
private final int maxRecordsPerCall;
private final GetRecordsRetrievalStrategy getRecordsRetrievalStrategy;

48
src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/GracefulShutdownContext.java
View file

@ -14,11 +14,9 @@
*/
package com.amazonaws.services.kinesis.clientlibrary.lib.worker;
import lombok.Data;
import java.util.Objects;
import java.util.concurrent.CountDownLatch;
@Data
class GracefulShutdownContext {
private final CountDownLatch shutdownCompleteLatch;
private final CountDownLatch notificationCompleteLatch;
@ -26,8 +24,52 @@ class GracefulShutdownContext {
static GracefulShutdownContext SHUTDOWN_ALREADY_COMPLETED = new GracefulShutdownContext(null, null, null);
public GracefulShutdownContext(CountDownLatch shutdownCompleteLatch, CountDownLatch notificationCompleteLatch, Worker worker) {
this.shutdownCompleteLatch = shutdownCompleteLatch;
this.notificationCompleteLatch = notificationCompleteLatch;
this.worker = worker;
Objects.requireNonNull(shutdownCompleteLatch);
Objects.requireNonNull(notificationCompleteLatch);
Objects.requireNonNull(worker);
}
boolean isShutdownAlreadyCompleted() {
return shutdownCompleteLatch == null && notificationCompleteLatch == null && worker == null;
}
public CountDownLatch getShutdownCompleteLatch() {
return shutdownCompleteLatch;
}
public CountDownLatch getNotificationCompleteLatch() {
return notificationCompleteLatch;
}
public Worker getWorker() {
return worker;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
GracefulShutdownContext that = (GracefulShutdownContext) o;
return Objects.equals(shutdownCompleteLatch, that.shutdownCompleteLatch) &&
Objects.equals(notificationCompleteLatch, that.notificationCompleteLatch) &&
Objects.equals(worker, that.worker);
}
@Override
public int hashCode() {
return Objects.hash(shutdownCompleteLatch, notificationCompleteLatch, worker);
}
@Override
public String toString() {
return "GracefulShutdownContext{" +
"shutdownCompleteLatch=" + shutdownCompleteLatch +
", notificationCompleteLatch=" + notificationCompleteLatch +
", worker=" + worker +
'}';
}
}

51
src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java
View file

@ -22,14 +22,11 @@ import org.apache.commons.lang3.Validate;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.regions.RegionUtils;
import com.amazonaws.services.kinesis.metrics.impl.MetricsHelper;
import com.amazonaws.services.kinesis.metrics.interfaces.IMetricsScope;
import com.amazonaws.services.kinesis.metrics.interfaces.MetricsLevel;
import com.google.common.collect.ImmutableSet;
import lombok.Getter;
/**
* Configuration for the Amazon Kinesis Client Library.
*/
@ -231,28 +228,14 @@ public class KinesisClientLibConfiguration {
private ShardPrioritization shardPrioritization;
private long shutdownGraceMillis;
@Getter
/* there should be getters for the following */
private Optional<Integer> timeoutInSeconds = Optional.empty();
@Getter
private Optional<Integer> retryGetRecordsInSeconds = Optional.empty();
@Getter
private Optional<Integer> maxGetRecordsThreadPool = Optional.empty();
@Getter
private int maxLeaseRenewalThreads = DEFAULT_MAX_LEASE_RENEWAL_THREADS;
@Getter
private RecordsFetcherFactory recordsFetcherFactory;
@Getter
private Optional<Long> logWarningForTaskAfterMillis = Optional.empty();
@Getter
private long listShardsBackoffTimeInMillis = DEFAULT_LIST_SHARDS_BACKOFF_TIME_IN_MILLIS;
@Getter
private int maxListShardsRetryAttempts = DEFAULT_MAX_LIST_SHARDS_RETRY_ATTEMPTS;
/**
@ -1416,4 +1399,36 @@ public class KinesisClientLibConfiguration {
this.maxListShardsRetryAttempts = maxListShardsRetryAttempts;
return this;
}
public Optional<Integer> getTimeoutInSeconds() {
return timeoutInSeconds;
}
public Optional<Integer> getRetryGetRecordsInSeconds() {
return retryGetRecordsInSeconds;
}
public Optional<Integer> getMaxGetRecordsThreadPool() {
return maxGetRecordsThreadPool;
}
public int getMaxLeaseRenewalThreads() {
return maxLeaseRenewalThreads;
}
public RecordsFetcherFactory getRecordsFetcherFactory() {
return recordsFetcherFactory;
}
public Optional<Long> getLogWarningForTaskAfterMillis() {
return logWarningForTaskAfterMillis;
}
public long getListShardsBackoffTimeInMillis() {
return listShardsBackoffTimeInMillis;
}
public int getMaxListShardsRetryAttempts() {
return maxListShardsRetryAttempts;
}
}

29
src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisDataFetcher.java
View file

@ -16,6 +16,7 @@ package com.amazonaws.services.kinesis.clientlibrary.lib.worker;
import java.util.Collections;
import java.util.Date;
import java.util.Objects;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
@ -31,8 +32,6 @@ import com.amazonaws.services.kinesis.model.ShardIteratorType;
import com.amazonaws.util.CollectionUtils;
import com.google.common.collect.Iterables;
import lombok.Data;
/**
* Used to get data from Amazon Kinesis. Tracks iterator state internally.
*/
@ -100,11 +99,15 @@ class KinesisDataFetcher {
}
};
@Data
class AdvancingResult implements DataFetcherResult {
final GetRecordsResult result;
public AdvancingResult(GetRecordsResult result) {
this.result = result;
Objects.requireNonNull(result);
}
@Override
public GetRecordsResult getResult() {
return result;
@ -126,6 +129,26 @@ class KinesisDataFetcher {
public boolean isShardEnd() {
return isShardEndReached;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
AdvancingResult that = (AdvancingResult) o;
return Objects.equals(result, that.result);
}
@Override
public int hashCode() {
return Objects.hash(result);
}
@Override
public String toString() {
return "AdvancingResult{" +
"result=" + result +
'}';
}
}
/**

27
src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PrefetchGetRecordsCache.java
View file

@ -17,6 +17,7 @@ package com.amazonaws.services.kinesis.clientlibrary.lib.worker;
import java.time.Duration;
import java.time.Instant;
import java.util.Objects;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
@ -31,9 +32,7 @@ import com.amazonaws.services.kinesis.metrics.interfaces.IMetricsFactory;
import com.amazonaws.services.kinesis.metrics.interfaces.MetricsLevel;
import com.amazonaws.services.kinesis.model.ExpiredIteratorException;
import com.amazonaws.services.kinesis.model.GetRecordsResult;
import lombok.NonNull;
import lombok.extern.apachecommons.CommonsLog;
import org.apache.commons.logging.impl.SimpleLog;
/**
* This is the prefetch caching class, this class spins up a thread if prefetching is enabled. That thread fetches the
@ -43,9 +42,10 @@ import lombok.extern.apachecommons.CommonsLog;
* be present in the cache across multiple GetRecordsResult object. If no data is available in the cache, the call from
* the record processor is blocked till records are retrieved from Kinesis.
*/
@CommonsLog
public class PrefetchGetRecordsCache implements GetRecordsCache {
private static final org.apache.commons.logging.Log log = org.apache.commons.logging.LogFactory.getLog(SimpleLog.class);
private static final String EXPIRED_ITERATOR_METRIC = "ExpiredIterator";
LinkedBlockingQueue<ProcessRecordsInput> getRecordsResultQueue;
private int maxPendingProcessRecordsInput;
private int maxByteSize;
@ -78,14 +78,16 @@ public class PrefetchGetRecordsCache implements GetRecordsCache {
* @param executorService Executor service for the cache
* @param idleMillisBetweenCalls maximum time to wait before dispatching the next get records call
*/
public PrefetchGetRecordsCache(final int maxPendingProcessRecordsInput, final int maxByteSize, final int maxRecordsCount,
public PrefetchGetRecordsCache(final int maxPendingProcessRecordsInput,
final int maxByteSize,
final int maxRecordsCount,
final int maxRecordsPerCall,
@NonNull final GetRecordsRetrievalStrategy getRecordsRetrievalStrategy,
@NonNull final ExecutorService executorService,
final GetRecordsRetrievalStrategy getRecordsRetrievalStrategy,
final ExecutorService executorService,
final long idleMillisBetweenCalls,
@NonNull final IMetricsFactory metricsFactory,
@NonNull final String operation,
@NonNull final String shardId) {
final IMetricsFactory metricsFactory,
final String operation,
final String shardId) {
this.getRecordsRetrievalStrategy = getRecordsRetrievalStrategy;
this.maxRecordsPerCall = maxRecordsPerCall;
this.maxPendingProcessRecordsInput = maxPendingProcessRecordsInput;
@ -101,6 +103,11 @@ public class PrefetchGetRecordsCache implements GetRecordsCache {
this.operation = operation;
this.dataFetcher = this.getRecordsRetrievalStrategy.getDataFetcher();
this.shardId = shardId;
Objects.requireNonNull(getRecordsRetrievalStrategy);
Objects.requireNonNull(executorService);
Objects.requireNonNull(metricsFactory);
Objects.requireNonNull(operation);
Objects.requireNonNull(shardId);
}
@Override

7
src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumer.java
View file

@ -31,8 +31,6 @@ import com.amazonaws.services.kinesis.leases.interfaces.ILeaseManager;
import com.amazonaws.services.kinesis.metrics.interfaces.IMetricsFactory;
import com.google.common.annotations.VisibleForTesting;
import lombok.Getter;
/**
* Responsible for consuming data records of a (specified) shard.
* The instance should be shutdown when we lose the primary responsibility for a shard.
@ -62,7 +60,6 @@ class ShardConsumer {
private long currentTaskSubmitTime;
private Future<TaskResult> future;
@Getter
private final GetRecordsCache getRecordsCache;
private static final GetRecordsRetrievalStrategy makeStrategy(KinesisDataFetcher dataFetcher,
@ -304,6 +301,10 @@ class ShardConsumer {
return skipShardSyncAtWorkerInitializationIfLeasesExist;
}
public GetRecordsCache getGetRecordsCache() {
return getRecordsCache;
}
private enum TaskOutcome {
SUCCESSFUL, END_OF_SHARD, NOT_COMPLETE, FAILURE
}

6
src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/SimpleRecordsFetcherFactory.java
View file

@ -18,11 +18,11 @@ import java.util.concurrent.Executors;
import com.amazonaws.services.kinesis.metrics.interfaces.IMetricsFactory;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.commons.logging.impl.SimpleLog;
import lombok.extern.apachecommons.CommonsLog;
@CommonsLog
public class SimpleRecordsFetcherFactory implements RecordsFetcherFactory {
private static final org.apache.commons.logging.Log log = org.apache.commons.logging.LogFactory.getLog(SimpleLog.class);
private int maxPendingProcessRecordsInput = 3;
private int maxByteSize = 8 * 1024 * 1024;
private int maxRecordsCount = 30000;

31
src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/SynchronousGetRecordsRetrievalStrategy.java
View file

@ -15,17 +15,20 @@
package com.amazonaws.services.kinesis.clientlibrary.lib.worker;
import com.amazonaws.services.kinesis.model.GetRecordsResult;
import lombok.Data;
import lombok.NonNull;
import java.util.Objects;
/**
*
*/
@Data
public class SynchronousGetRecordsRetrievalStrategy implements GetRecordsRetrievalStrategy {
@NonNull
private final KinesisDataFetcher dataFetcher;
public SynchronousGetRecordsRetrievalStrategy(KinesisDataFetcher dataFetcher) {
this.dataFetcher = dataFetcher;
Objects.requireNonNull(dataFetcher);
}
@Override
public GetRecordsResult getRecords(final int maxRecords) {
return dataFetcher.getRecords(maxRecords).accept();
@ -47,4 +50,24 @@ public class SynchronousGetRecordsRetrievalStrategy implements GetRecordsRetriev
public KinesisDataFetcher getDataFetcher() {
return dataFetcher;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
SynchronousGetRecordsRetrievalStrategy that = (SynchronousGetRecordsRetrievalStrategy) o;
return Objects.equals(dataFetcher, that.dataFetcher);
}
@Override
public int hashCode() {
return Objects.hash(dataFetcher);
}
@Override
public String toString() {
return "SynchronousGetRecordsRetrievalStrategy{" +
"dataFetcher=" + dataFetcher +
'}';
}
}

12
src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ThrottlingReporter.java
View file

@ -14,20 +14,22 @@
*/
package com.amazonaws.services.kinesis.clientlibrary.lib.worker;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.extern.apachecommons.CommonsLog;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.impl.SimpleLog;
@RequiredArgsConstructor
@CommonsLog
class ThrottlingReporter {
private static final org.apache.commons.logging.Log log = org.apache.commons.logging.LogFactory.getLog(SimpleLog.class);
private final int maxConsecutiveWarnThrottles;
private final String shardId;
private int consecutiveThrottles = 0;
public ThrottlingReporter(int maxConsecutiveWarnThrottles, String shardId) {
this.maxConsecutiveWarnThrottles = maxConsecutiveWarnThrottles;
this.shardId = shardId;
}
void throttled() {
consecutiveThrottles++;
String message = "Shard '" + shardId + "' has been throttled "

76
src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java
View file

@ -38,7 +38,6 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.amazonaws.AmazonWebServiceClient;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.client.builder.AwsClientBuilder;
@ -70,10 +69,6 @@ import com.amazonaws.services.kinesis.metrics.interfaces.MetricsLevel;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import lombok.NonNull;
import lombok.Setter;
import lombok.experimental.Accessors;
/**
* Worker is the high level class that Kinesis applications use to start processing data. It initializes and oversees
* different components (e.g. syncing shard and lease information, tracking shard assignments, and processing data from
@ -1054,36 +1049,17 @@ public class Worker implements Runnable {
*/
public static class Builder {
// make setters for each field minus the "set" part
private IRecordProcessorFactory recordProcessorFactory;
@Setter
@Accessors(fluent = true)
private KinesisClientLibConfiguration config;
@Setter
@Accessors(fluent = true)
private AmazonKinesis kinesisClient;
@Setter
@Accessors(fluent = true)
private AmazonDynamoDB dynamoDBClient;
@Setter
@Accessors(fluent = true)
private AmazonCloudWatch cloudWatchClient;
@Setter
@Accessors(fluent = true)
private IMetricsFactory metricsFactory;
@Setter
@Accessors(fluent = true)
private ILeaseManager<KinesisClientLease> leaseManager;
@Setter
@Accessors(fluent = true)
private ExecutorService execService;
@Setter
@Accessors(fluent = true)
private ShardPrioritization shardPrioritization;
@Setter
@Accessors(fluent = true)
private IKinesisProxy kinesisProxy;
@Setter
@Accessors(fluent = true)
private WorkerStateChangeListener workerStateChangeListener;
@VisibleForTesting
@ -1256,5 +1232,55 @@ public class Worker implements Runnable {
}
return builder.build();
}
public Builder config(KinesisClientLibConfiguration config) {
this.config = config;
return this;
}
public Builder kinesisClient(AmazonKinesis kinesisClient) {
this.kinesisClient = kinesisClient;
return this;
}
public Builder dynamoDBClient(AmazonDynamoDB dynamoDBClient) {
this.dynamoDBClient = dynamoDBClient;
return this;
}
public Builder cloudWatchClient(AmazonCloudWatch cloudWatchClient) {
this.cloudWatchClient = cloudWatchClient;
return this;
}
public Builder metricsFactory(IMetricsFactory metricsFactory) {
this.metricsFactory = metricsFactory;
return this;
}
public Builder leaseManager(ILeaseManager<KinesisClientLease> leaseManager) {
this.leaseManager = leaseManager;
return this;
}
public Builder execService(ExecutorService execService) {
this.execService = execService;
return this;
}
public Builder shardPrioritization(ShardPrioritization shardPrioritization) {
this.shardPrioritization = shardPrioritization;
return this;
}
public Builder kinesisProxy(IKinesisProxy kinesisProxy) {
this.kinesisProxy = kinesisProxy;
return this;
}
public Builder workerStateChangeListener(WorkerStateChangeListener workerStateChangeListener) {
this.workerStateChangeListener = workerStateChangeListener;
return this;
}
}
}

85
src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxy.java
View file

@ -18,13 +18,7 @@ import java.nio.ByteBuffer;
import java.time.Duration;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Date;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
import java.util.stream.Collectors;
@ -56,11 +50,6 @@ import com.amazonaws.services.kinesis.model.Shard;
import com.amazonaws.services.kinesis.model.ShardIteratorType;
import com.amazonaws.services.kinesis.model.StreamStatus;
import lombok.AccessLevel;
import lombok.Data;
import lombok.Getter;
import lombok.Setter;
/**
* Kinesis proxy - used to make calls to Amazon Kinesis (e.g. fetch data records and list of shards).
*/
@ -82,14 +71,9 @@ public class KinesisProxy implements IKinesisProxyExtended {
private ShardIterationState shardIterationState = null;
@Setter(AccessLevel.PACKAGE)
private volatile Map<String, Shard> cachedShardMap = null;
@Setter(AccessLevel.PACKAGE)
@Getter(AccessLevel.PACKAGE)
private volatile Instant lastCacheUpdateTime = null;
@Setter(AccessLevel.PACKAGE)
@Getter(AccessLevel.PACKAGE)
private AtomicInteger cacheMisses = new AtomicInteger(0);
volatile Map<String, Shard> cachedShardMap = null;
volatile Instant lastCacheUpdateTime = null;
AtomicInteger cacheMisses = new AtomicInteger(0);
private final String streamName;
@ -575,7 +559,26 @@ public class KinesisProxy implements IKinesisProxyExtended {
return response;
}
@Data
public void setCachedShardMap(Map<String, Shard> cachedShardMap) {
this.cachedShardMap = cachedShardMap;
}
public void setLastCacheUpdateTime(Instant lastCacheUpdateTime) {
this.lastCacheUpdateTime = lastCacheUpdateTime;
}
public void setCacheMisses(AtomicInteger cacheMisses) {
this.cacheMisses = cacheMisses;
}
public Instant getLastCacheUpdateTime() {
return lastCacheUpdateTime;
}
public AtomicInteger getCacheMisses() {
return cacheMisses;
}
static class ShardIterationState {
private List<Shard> shards;
@ -595,6 +598,46 @@ public class KinesisProxy implements IKinesisProxyExtended {
lastShardId = lastShard.getShardId();
}
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
ShardIterationState that = (ShardIterationState) o;
return Objects.equals(shards, that.shards) &&
Objects.equals(lastShardId, that.lastShardId);
}
@Override
public int hashCode() {
return Objects.hash(shards, lastShardId);
}
public List<Shard> getShards() {
return shards;
}
public void setShards(List<Shard> shards) {
Objects.requireNonNull(shards);
this.shards = shards;
}
public String getLastShardId() {
return lastShardId;
}
public void setLastShardId(String lastShardId) {
Objects.requireNonNull(lastShardId);
this.lastShardId = lastShardId;
}
@Override
public String toString() {
return "ShardIterationState{" +
"shards=" + shards +
", lastShardId='" + lastShardId + '\'' +
'}';
}
}
}

12
src/main/java/com/amazonaws/services/kinesis/clientlibrary/types/ProcessRecordsInput.java
View file

@ -21,17 +21,13 @@ import java.util.List;
import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer;
import com.amazonaws.services.kinesis.model.Record;
import lombok.Getter;
/**
* Container for the parameters to the IRecordProcessor's
* {@link com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessor#processRecords(
* ProcessRecordsInput processRecordsInput) processRecords} method.
*/
public class ProcessRecordsInput {
@Getter
private Instant cacheEntryTime;
@Getter
private Instant cacheExitTime;
private List<Record> records;
private IRecordProcessorCheckpointer checkpointer;
@ -121,4 +117,12 @@ public class ProcessRecordsInput {
}
return Duration.between(cacheEntryTime, cacheExitTime);
}
public Instant getCacheEntryTime() {
return cacheEntryTime;
}
public Instant getCacheExitTime() {
return cacheExitTime;
}
}

4
src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangProtocol.java
View file

@ -27,7 +27,7 @@ import com.amazonaws.services.kinesis.multilang.messages.ProcessRecordsMessage;
import com.amazonaws.services.kinesis.multilang.messages.ShutdownMessage;
import com.amazonaws.services.kinesis.multilang.messages.ShutdownRequestedMessage;
import com.amazonaws.services.kinesis.multilang.messages.StatusMessage;
import lombok.extern.apachecommons.CommonsLog;
import org.apache.commons.logging.impl.SimpleLog;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
@ -38,8 +38,8 @@ import java.util.concurrent.TimeoutException;
/**
* An implementation of the multi language protocol.
*/
@CommonsLog
class MultiLangProtocol {
private static final org.apache.commons.logging.Log log = org.apache.commons.logging.LogFactory.getLog(SimpleLog.class);
private MessageReader messageReader;
private MessageWriter messageWriter;

28
src/main/java/com/amazonaws/services/kinesis/multilang/messages/CheckpointMessage.java
View file

@ -14,16 +14,11 @@
*/
package com.amazonaws.services.kinesis.multilang.messages;
import lombok.Getter;
import lombok.Setter;
/**
* A checkpoint message is sent by the client's subprocess to indicate to the kcl processor that it should attempt to
* checkpoint. The processor sends back a checkpoint message as an acknowledgement that it attempted to checkpoint along
* with an error message which corresponds to the names of exceptions that a checkpointer can throw.
*/
@Getter
@Setter
public class CheckpointMessage extends Message {
/**
* The name used for the action field in {@link Message}.
@ -66,4 +61,27 @@ public class CheckpointMessage extends Message {
}
}
public String getSequenceNumber() {
return sequenceNumber;
}
public void setSequenceNumber(String sequenceNumber) {
this.sequenceNumber = sequenceNumber;
}
public Long getSubSequenceNumber() {
return subSequenceNumber;
}
public void setSubSequenceNumber(Long subSequenceNumber) {
this.subSequenceNumber = subSequenceNumber;
}
public String getError() {
return error;
}
public void setError(String error) {
this.error = error;
}
}

27
src/main/java/com/amazonaws/services/kinesis/multilang/messages/InitializeMessage.java
View file

@ -15,14 +15,10 @@
package com.amazonaws.services.kinesis.multilang.messages;
import com.amazonaws.services.kinesis.clientlibrary.types.InitializationInput;
import lombok.Getter;
import lombok.Setter;
/**
* An initialize message is sent to the client's subprocess to indicate that it should perform its initialization steps.
*/
@Getter
@Setter
public class InitializeMessage extends Message {
/**
* The name used for the action field in {@link Message}.
@ -59,4 +55,27 @@ public class InitializeMessage extends Message {
}
public String getShardId() {
return shardId;
}
public void setShardId(String shardId) {
this.shardId = shardId;
}
public String getSequenceNumber() {
return sequenceNumber;
}
public void setSequenceNumber(String sequenceNumber) {
this.sequenceNumber = sequenceNumber;
}
public Long getSubSequenceNumber() {
return subSequenceNumber;
}
public void setSubSequenceNumber(Long subSequenceNumber) {
this.subSequenceNumber = subSequenceNumber;
}
}

44
src/main/java/com/amazonaws/services/kinesis/multilang/messages/JsonFriendlyRecord.java
View file

@ -20,15 +20,10 @@ import com.amazonaws.services.kinesis.clientlibrary.types.UserRecord;
import com.amazonaws.services.kinesis.model.Record;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Getter;
import lombok.Setter;
/**
* Class for encoding Record objects to json. Needed because Records have byte buffers for their data field which causes
* problems for the json library we're using.
*/
@Getter
@Setter
public class JsonFriendlyRecord {
private byte[] data;
private String partitionKey;
@ -66,4 +61,43 @@ public class JsonFriendlyRecord {
return ACTION;
}
public byte[] getData() {
return data;
}
public void setData(byte[] data) {
this.data = data;
}
public String getPartitionKey() {
return partitionKey;
}
public void setPartitionKey(String partitionKey) {
this.partitionKey = partitionKey;
}
public String getSequenceNumber() {
return sequenceNumber;
}
public void setSequenceNumber(String sequenceNumber) {
this.sequenceNumber = sequenceNumber;
}
public Date getApproximateArrivalTimestamp() {
return approximateArrivalTimestamp;
}
public void setApproximateArrivalTimestamp(Date approximateArrivalTimestamp) {
this.approximateArrivalTimestamp = approximateArrivalTimestamp;
}
public Long getSubSequenceNumber() {
return subSequenceNumber;
}
public void setSubSequenceNumber(Long subSequenceNumber) {
this.subSequenceNumber = subSequenceNumber;
}
}

20
src/main/java/com/amazonaws/services/kinesis/multilang/messages/ProcessRecordsMessage.java
View file

@ -19,14 +19,10 @@ import java.util.List;
import com.amazonaws.services.kinesis.clientlibrary.types.ProcessRecordsInput;
import com.amazonaws.services.kinesis.model.Record;
import lombok.Getter;
import lombok.Setter;
/**
* A message to indicate to the client's process that it should process a list of records.
*/
@Getter
@Setter
public class ProcessRecordsMessage extends Message {
/**
* The name used for the action field in {@link Message}.
@ -59,4 +55,20 @@ public class ProcessRecordsMessage extends Message {
}
this.setRecords(recordMessages);
}
public List<JsonFriendlyRecord> getRecords() {
return records;
}
public void setRecords(List<JsonFriendlyRecord> records) {
this.records = records;
}
public Long getMillisBehindLatest() {
return millisBehindLatest;
}
public void setMillisBehindLatest(Long millisBehindLatest) {
this.millisBehindLatest = millisBehindLatest;
}
}

5
src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PrefetchGetRecordsCacheIntegrationTest.java
View file

@ -33,6 +33,7 @@ import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.apache.commons.logging.impl.SimpleLog;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@ -49,14 +50,12 @@ import com.amazonaws.services.kinesis.model.ExpiredIteratorException;
import com.amazonaws.services.kinesis.model.GetRecordsResult;
import com.amazonaws.services.kinesis.model.Record;
import lombok.extern.apachecommons.CommonsLog;
/**
* These are the integration tests for the PrefetchGetRecordsCache class.
*/
@RunWith(MockitoJUnitRunner.class)
@CommonsLog
public class PrefetchGetRecordsCacheIntegrationTest {
private static final org.apache.commons.logging.Log log = org.apache.commons.logging.LogFactory.getLog(SimpleLog.class);
private static final int MAX_SIZE = 3;
private static final int MAX_BYTE_SIZE = 5 * 1024 * 1024;
private static final int MAX_RECORDS_COUNT = 30_000;

28
src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/WorkerTest.java
View file

@ -22,13 +22,7 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyInt;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Matchers.anyString;
import static org.mockito.Matchers.argThat;
import static org.mockito.Matchers.eq;
import static org.mockito.Matchers.same;
import static org.mockito.Matchers.*;
import static org.mockito.Mockito.atLeast;
import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.doAnswer;
@ -66,6 +60,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import com.amazonaws.auth.AWSCredentialsProvider;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.hamcrest.Condition;
@ -73,6 +68,7 @@ import org.hamcrest.Description;
import org.hamcrest.Matcher;
import org.hamcrest.TypeSafeDiagnosingMatcher;
import org.hamcrest.TypeSafeMatcher;
import org.hamcrest.internal.ReflectiveTypeFinder;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@ -126,8 +122,6 @@ import com.amazonaws.services.kinesis.model.Shard;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import lombok.RequiredArgsConstructor;
/**
* Unit tests of Worker.
*/
@ -1715,8 +1709,9 @@ public class WorkerTest {
assertTrue(builder.getDynamoDBClient() instanceof AmazonDynamoDB);
assertTrue(builder.getCloudWatchClient() instanceof AmazonCloudWatch);
verify(builder, times(3)).createClient(
builderCaptor.capture(), eq(null), any(ClientConfiguration.class), eq(null), eq(null));
// verify(builder, times(3)).createClient(
// builderCaptor.capture(), eq(null), any(ClientConfiguration.class), eq(null), eq(null));
// builderCaptor.capture(), null, any(ClientConfiguration.class), null, null);
builderCaptor.getAllValues().forEach(clientBuilder -> {
assertTrue(clientBuilder.getRegion().equals(Regions.US_EAST_1.getName()));
@ -1735,8 +1730,8 @@ public class WorkerTest {
builder.recordProcessorFactory(recordProcessorFactory).config(config).build();
verify(builder, times(3)).createClient(
builderCaptor.capture(), eq(null), any(ClientConfiguration.class), eq(null), eq(region));
// verify(builder, times(3)).createClient(
// builderCaptor.capture(), eq(null), any(ClientConfiguration.class), eq(null), eq(region));
builderCaptor.getAllValues().forEach(clientBuilder -> {
assertTrue(clientBuilder.getRegion().equals(region));
});
@ -1916,7 +1911,6 @@ public class WorkerTest {
}
}
@RequiredArgsConstructor
private static class ReflectionFieldMatcher<T extends ITask>
extends TypeSafeDiagnosingMatcher<MetricsCollectingTaskDecorator> {
@ -1924,6 +1918,12 @@ public class WorkerTest {
private final String fieldName;
private final Matcher<?> fieldMatcher;
public ReflectionFieldMatcher(Class<T> itemClass, String fieldName, Matcher<?> fieldMatcher) {
this.itemClass = itemClass;
this.fieldName = fieldName;
this.fieldMatcher = fieldMatcher;
}
@Override
protected boolean matchesSafely(MetricsCollectingTaskDecorator item, Description mismatchDescription) {
if (item.getOther() == null) {

16
src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxyTest.java
View file

@ -36,13 +36,7 @@ import static org.mockito.Mockito.when;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
import java.util.stream.Collectors;
@ -76,8 +70,6 @@ import com.amazonaws.services.kinesis.model.ShardIteratorType;
import com.amazonaws.services.kinesis.model.StreamDescription;
import com.amazonaws.services.kinesis.model.StreamStatus;
import lombok.AllArgsConstructor;
@RunWith(MockitoJUnitRunner.class)
public class KinesisProxyTest {
private static final String TEST_STRING = "TestString";
@ -569,11 +561,15 @@ public class KinesisProxyTest {
return new ListShardsRequestMatcher(null, nextToken);
}
@AllArgsConstructor
private static class ListShardsRequestMatcher extends TypeSafeDiagnosingMatcher<ListShardsRequest> {
private final String shardId;
private final String nextToken;
public ListShardsRequestMatcher(String shardId, String nextToken) {
this.shardId = shardId;
this.nextToken = nextToken;
}
@Override
protected boolean matchesSafely(final ListShardsRequest listShardsRequest, final Description description) {
if (shardId == null) {