diff --git a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangDaemon.java b/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangDaemon.java index a99bd4bf..57414fdf 100644 --- a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangDaemon.java +++ b/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangDaemon.java @@ -26,7 +26,7 @@ import java.util.concurrent.TimeoutException; import lombok.extern.slf4j.Slf4j; import software.amazon.kinesis.coordinator.KinesisClientLibConfiguration; import software.amazon.kinesis.coordinator.Scheduler; -import software.amazon.kinesis.processor.IRecordProcessorFactory; +import software.amazon.kinesis.processor.RecordProcessorFactory; /** * Main app that launches the scheduler that runs the multi-language record processor. @@ -73,7 +73,7 @@ public class MultiLangDaemon implements Callable { this(buildWorker(recordProcessorFactory, configuration, workerThreadPool)); } - private static Scheduler buildWorker(IRecordProcessorFactory recordProcessorFactory, + private static Scheduler buildWorker(RecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration configuration, ExecutorService workerThreadPool) { return null; } diff --git a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangRecordProcessor.java b/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangRecordProcessor.java index b4e9ba82..8b050dd7 100644 --- a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangRecordProcessor.java +++ b/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangRecordProcessor.java @@ -21,8 +21,8 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import software.amazon.kinesis.processor.IRecordProcessorCheckpointer; -import software.amazon.kinesis.processor.IRecordProcessor; -import software.amazon.kinesis.processor.IShutdownNotificationAware; +import software.amazon.kinesis.processor.RecordProcessor; +import software.amazon.kinesis.processor.ShutdownNotificationAware; import software.amazon.kinesis.coordinator.KinesisClientLibConfiguration; import software.amazon.kinesis.lifecycle.InitializationInput; import software.amazon.kinesis.lifecycle.ProcessRecordsInput; @@ -39,7 +39,7 @@ import lombok.extern.slf4j.Slf4j; * called. */ @Slf4j -public class MultiLangRecordProcessor implements IRecordProcessor, IShutdownNotificationAware { +public class MultiLangRecordProcessor implements RecordProcessor, ShutdownNotificationAware { private static final int EXIT_VALUE = 1; /** Whether or not record processor initialization is successful. Defaults to false. */ diff --git a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangRecordProcessorFactory.java b/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangRecordProcessorFactory.java index da423c3d..6a34fa37 100644 --- a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangRecordProcessorFactory.java +++ b/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangRecordProcessorFactory.java @@ -16,8 +16,8 @@ package com.amazonaws.services.kinesis.multilang; import java.util.concurrent.ExecutorService; -import software.amazon.kinesis.processor.IRecordProcessor; -import software.amazon.kinesis.processor.IRecordProcessorFactory; +import software.amazon.kinesis.processor.RecordProcessor; +import software.amazon.kinesis.processor.RecordProcessorFactory; import software.amazon.kinesis.coordinator.KinesisClientLibConfiguration; import com.fasterxml.jackson.databind.ObjectMapper; @@ -27,7 +27,7 @@ import lombok.extern.slf4j.Slf4j; * Creates {@link MultiLangRecordProcessor}'s. */ @Slf4j -public class MultiLangRecordProcessorFactory implements IRecordProcessorFactory { +public class MultiLangRecordProcessorFactory implements RecordProcessorFactory { private static final String COMMAND_DELIMETER_REGEX = " +"; private final String command; @@ -63,7 +63,7 @@ public class MultiLangRecordProcessorFactory implements IRecordProcessorFactory } @Override - public IRecordProcessor createProcessor() { + public RecordProcessor createProcessor() { log.debug("Creating new record processor for client executable: {}", command); /* * Giving ProcessBuilder the command as an array of Strings allows users to specify command line arguments. diff --git a/amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/StreamingRecordProcessorFactoryTest.java b/amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/StreamingRecordProcessorFactoryTest.java index 3b7d447d..513ba2fb 100644 --- a/amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/StreamingRecordProcessorFactoryTest.java +++ b/amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/StreamingRecordProcessorFactoryTest.java @@ -18,7 +18,7 @@ import software.amazon.kinesis.coordinator.KinesisClientLibConfiguration; import org.junit.Assert; import org.junit.Test; -import software.amazon.kinesis.processor.IRecordProcessor; +import software.amazon.kinesis.processor.RecordProcessor; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; @@ -32,7 +32,7 @@ public class StreamingRecordProcessorFactoryTest { @Test public void createProcessorTest() { MultiLangRecordProcessorFactory factory = new MultiLangRecordProcessorFactory("somecommand", null, configuration); - IRecordProcessor processor = factory.createProcessor(); + RecordProcessor processor = factory.createProcessor(); Assert.assertEquals("Should have constructed a StreamingRecordProcessor", MultiLangRecordProcessor.class, processor.getClass()); diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/Checkpoint.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/Checkpoint.java index f1681d57..df59e1d3 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/Checkpoint.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/Checkpoint.java @@ -15,12 +15,14 @@ package software.amazon.kinesis.checkpoint; import lombok.Data; +import lombok.experimental.Accessors; import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; /** * A class encapsulating the 2 pieces of state stored in a checkpoint. */ @Data +@Accessors(fluent = true) public class Checkpoint { private final ExtendedSequenceNumber checkpoint; private final ExtendedSequenceNumber pendingCheckpoint; diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/CheckpointConfig.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/CheckpointConfig.java index 2c702927..5fda7cf5 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/CheckpointConfig.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/CheckpointConfig.java @@ -16,13 +16,15 @@ package software.amazon.kinesis.checkpoint; import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; + import lombok.Data; import lombok.NonNull; import lombok.experimental.Accessors; -import software.amazon.kinesis.coordinator.RecordProcessorCheckpointer; import software.amazon.kinesis.leases.ILeaseManager; +import software.amazon.kinesis.leases.KinesisClientLease; import software.amazon.kinesis.leases.KinesisClientLeaseManager; -import software.amazon.kinesis.leases.LeaseManagementConfig; +import software.amazon.kinesis.leases.KinesisClientLibLeaseCoordinator; +import software.amazon.kinesis.leases.LeaseCoordinator; import software.amazon.kinesis.metrics.IMetricsFactory; import software.amazon.kinesis.metrics.NullMetricsFactory; @@ -53,7 +55,7 @@ public class CheckpointConfig { private long failoverTimeMillis = 10000L; - private ILeaseManager leaseManager; + private ILeaseManager leaseManager; private int maxLeasesForWorker = Integer.MAX_VALUE; @@ -67,7 +69,9 @@ public class CheckpointConfig { private long epsilonMillis = 25L; - public ILeaseManager leaseManager() { + private LeaseCoordinator leaseCoordinator; + + public ILeaseManager leaseManager() { if (leaseManager == null) { leaseManager = new KinesisClientLeaseManager(tableName, amazonDynamoDB, consistentReads); } @@ -76,7 +80,14 @@ public class CheckpointConfig { public CheckpointFactory checkpointFactory() { if (checkpointFactory == null) { - checkpointFactory = new DynamoDBCheckpointFactory(leaseManager(), + checkpointFactory = new DynamoDBCheckpointFactory(leaseCoordinator(), leaseManager(), metricsFactory()); + } + return checkpointFactory; + } + + public LeaseCoordinator leaseCoordinator() { + if (leaseCoordinator == null) { + leaseCoordinator = new KinesisClientLibLeaseCoordinator(leaseManager(), workerIdentifier(), failoverTimeMillis(), epsilonMillis(), @@ -85,6 +96,6 @@ public class CheckpointConfig { maxLeaseRenewalThreads(), metricsFactory()); } - return checkpointFactory; + return leaseCoordinator; } } diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/CheckpointFactory.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/CheckpointFactory.java index 693119ba..51c2cc26 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/CheckpointFactory.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/CheckpointFactory.java @@ -16,11 +16,11 @@ package software.amazon.kinesis.checkpoint; import software.amazon.kinesis.leases.KinesisClientLibLeaseCoordinator; -import software.amazon.kinesis.processor.ICheckpoint; +import software.amazon.kinesis.processor.Checkpointer; /** * */ public interface CheckpointFactory { - ICheckpoint createCheckpoint(KinesisClientLibLeaseCoordinator leaseCoordinator); + Checkpointer createCheckpoint(KinesisClientLibLeaseCoordinator leaseCoordinator); } diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/DoesNothingPreparedCheckpointer.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/DoesNothingPreparedCheckpointer.java index e26386e1..82135ff4 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/DoesNothingPreparedCheckpointer.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/DoesNothingPreparedCheckpointer.java @@ -48,7 +48,7 @@ public class DoesNothingPreparedCheckpointer implements IPreparedCheckpointer { * {@inheritDoc} */ @Override - public ExtendedSequenceNumber getPendingCheckpoint() { + public ExtendedSequenceNumber pendingCheckpoint() { return sequenceNumber; } diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/DynamoDBCheckpointFactory.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/DynamoDBCheckpointFactory.java index 0cbfdaa9..9924d0ac 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/DynamoDBCheckpointFactory.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/DynamoDBCheckpointFactory.java @@ -18,9 +18,10 @@ package software.amazon.kinesis.checkpoint; import lombok.Data; import lombok.NonNull; import software.amazon.kinesis.leases.ILeaseManager; -import software.amazon.kinesis.leases.KinesisClientLibLeaseCoordinator; +import software.amazon.kinesis.leases.KinesisClientLease; +import software.amazon.kinesis.leases.LeaseCoordinator; import software.amazon.kinesis.metrics.IMetricsFactory; -import software.amazon.kinesis.processor.ICheckpoint; +import software.amazon.kinesis.processor.Checkpointer; /** * @@ -28,19 +29,15 @@ import software.amazon.kinesis.processor.ICheckpoint; @Data public class DynamoDBCheckpointFactory implements CheckpointFactory { @NonNull - private final ILeaseManager leaseManager; + private final LeaseCoordinator leaseLeaseCoordinator; @NonNull - private final String workerIdentifier; - private final long failoverTimeMillis; - private final long epsilonMillis; - private final int maxLeasesForWorker; - private final int maxLeasesToStealAtOneTime; - private final int maxLeaseRenewalThreads; + private final ILeaseManager leaseManager; @NonNull private final IMetricsFactory metricsFactory; @Override - public ICheckpoint createCheckpoint(KinesisClientLibLeaseCoordinator leaseCoordinator) { - return leaseCoordinator; + public Checkpointer createCheckpoint() { + return new DynamoDBCheckpointer(leaseLeaseCoordinator, leaseManager, metricsFactory); } + } diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/DynamoDBCheckpointer.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/DynamoDBCheckpointer.java new file mode 100644 index 00000000..59c0a21a --- /dev/null +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/DynamoDBCheckpointer.java @@ -0,0 +1,146 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.kinesis.checkpoint; + +import java.util.Objects; +import java.util.UUID; + +import com.amazonaws.services.kinesis.clientlibrary.exceptions.KinesisClientLibDependencyException; +import com.amazonaws.services.kinesis.clientlibrary.exceptions.KinesisClientLibException; +import com.amazonaws.services.kinesis.clientlibrary.exceptions.ShutdownException; +import com.amazonaws.services.kinesis.clientlibrary.exceptions.ThrottlingException; +import com.amazonaws.services.kinesis.clientlibrary.exceptions.internal.KinesisClientLibIOException; +import com.google.common.annotations.VisibleForTesting; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import software.amazon.kinesis.leases.ILeaseManager; +import software.amazon.kinesis.leases.KinesisClientLease; +import software.amazon.kinesis.leases.LeaseCoordinator; +import software.amazon.kinesis.leases.exceptions.DependencyException; +import software.amazon.kinesis.leases.exceptions.InvalidStateException; +import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException; +import software.amazon.kinesis.metrics.IMetricsFactory; +import software.amazon.kinesis.processor.Checkpointer; +import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; + +/** + * + */ +@RequiredArgsConstructor +@Slf4j +public class DynamoDBCheckpointer implements Checkpointer { + @NonNull + private final LeaseCoordinator leaseCoordinator; + @NonNull + private final ILeaseManager leaseManager; + @NonNull + private final IMetricsFactory metricsFactory; + + @Override + public void setCheckpoint(final String shardId, final ExtendedSequenceNumber checkpointValue, + final String concurrencyToken) throws KinesisClientLibException { + try { + boolean wasSuccessful = setCheckpoint(shardId, checkpointValue, UUID.fromString(concurrencyToken)); + if (!wasSuccessful) { + throw new ShutdownException("Can't update checkpoint - instance doesn't hold the lease for this shard"); + } + } catch (ProvisionedThroughputException e) { + throw new ThrottlingException("Got throttled while updating checkpoint.", e); + } catch (InvalidStateException e) { + String message = "Unable to save checkpoint for shardId " + shardId; + log.error(message, e); + throw new com.amazonaws.services.kinesis.clientlibrary.exceptions.InvalidStateException(message, e); + } catch (DependencyException e) { + throw new KinesisClientLibDependencyException("Unable to save checkpoint for shardId " + shardId, e); + } + } + + @Override + public ExtendedSequenceNumber getCheckpoint(final String shardId) throws KinesisClientLibException { + try { + return leaseManager.getLease(shardId).getCheckpoint(); + } catch (DependencyException | InvalidStateException | ProvisionedThroughputException e) { + String message = "Unable to fetch checkpoint for shardId " + shardId; + log.error(message, e); + throw new KinesisClientLibIOException(message, e); + } + } + + @Override + public Checkpoint getCheckpointObject(final String shardId) throws KinesisClientLibException { + try { + KinesisClientLease lease = leaseManager.getLease(shardId); + return new Checkpoint(lease.getCheckpoint(), lease.getPendingCheckpoint()); + } catch (DependencyException | InvalidStateException | ProvisionedThroughputException e) { + String message = "Unable to fetch checkpoint for shardId " + shardId; + log.error(message, e); + throw new KinesisClientLibIOException(message, e); + } + } + + @Override + public void prepareCheckpoint(final String shardId, final ExtendedSequenceNumber pendingCheckpoint, + final String concurrencyToken) throws KinesisClientLibException { + try { + boolean wasSuccessful = + prepareCheckpoint(shardId, pendingCheckpoint, UUID.fromString(concurrencyToken)); + if (!wasSuccessful) { + throw new ShutdownException( + "Can't prepare checkpoint - instance doesn't hold the lease for this shard"); + } + } catch (ProvisionedThroughputException e) { + throw new ThrottlingException("Got throttled while preparing checkpoint.", e); + } catch (InvalidStateException e) { + String message = "Unable to prepare checkpoint for shardId " + shardId; + log.error(message, e); + throw new com.amazonaws.services.kinesis.clientlibrary.exceptions.InvalidStateException(message, e); + } catch (DependencyException e) { + throw new KinesisClientLibDependencyException("Unable to prepare checkpoint for shardId " + shardId, e); + } + } + + @VisibleForTesting + public boolean setCheckpoint(String shardId, ExtendedSequenceNumber checkpoint, UUID concurrencyToken) + throws DependencyException, InvalidStateException, ProvisionedThroughputException { + KinesisClientLease lease = leaseCoordinator.getCurrentlyHeldLease(shardId); + if (lease == null) { + log.info("Worker {} could not update checkpoint for shard {} because it does not hold the lease", + leaseCoordinator.getWorkerIdentifier(), shardId); + return false; + } + + lease.setCheckpoint(checkpoint); + lease.setPendingCheckpoint(null); + lease.setOwnerSwitchesSinceCheckpoint(0L); + + return leaseCoordinator.updateLease(lease, concurrencyToken); + } + + boolean prepareCheckpoint(String shardId, ExtendedSequenceNumber pendingCheckpoint, UUID concurrencyToken) + throws DependencyException, InvalidStateException, ProvisionedThroughputException { + KinesisClientLease lease = leaseCoordinator.getCurrentlyHeldLease(shardId); + if (lease == null) { + log.info("Worker {} could not prepare checkpoint for shard {} because it does not hold the lease", + leaseCoordinator.getWorkerIdentifier(), shardId); + return false; + } + + lease.setPendingCheckpoint(Objects.requireNonNull(pendingCheckpoint, "pendingCheckpoint should not be null")); + return leaseCoordinator.updateLease(lease, concurrencyToken); + } +} diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/PreparedCheckpointer.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/PreparedCheckpointer.java index 91c92c52..ee504494 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/PreparedCheckpointer.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/PreparedCheckpointer.java @@ -48,7 +48,7 @@ public class PreparedCheckpointer implements IPreparedCheckpointer { * {@inheritDoc} */ @Override - public ExtendedSequenceNumber getPendingCheckpoint() { + public ExtendedSequenceNumber pendingCheckpoint() { return pendingCheckpointSequenceNumber; } diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/RecordProcessorCheckpointer.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/RecordProcessorCheckpointer.java similarity index 99% rename from amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/RecordProcessorCheckpointer.java rename to amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/RecordProcessorCheckpointer.java index 014d9cd2..cdf819bb 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/RecordProcessorCheckpointer.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/RecordProcessorCheckpointer.java @@ -12,7 +12,7 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package software.amazon.kinesis.coordinator; +package software.amazon.kinesis.checkpoint; import com.amazonaws.services.kinesis.clientlibrary.exceptions.InvalidStateException; import com.amazonaws.services.kinesis.clientlibrary.exceptions.KinesisClientLibDependencyException; @@ -32,7 +32,7 @@ import software.amazon.kinesis.leases.ShardInfo; import software.amazon.kinesis.metrics.IMetricsFactory; import software.amazon.kinesis.metrics.MetricsHelper; import software.amazon.kinesis.metrics.ThreadSafeMetricsDelegatingScope; -import software.amazon.kinesis.processor.ICheckpoint; +import software.amazon.kinesis.processor.Checkpointer; import software.amazon.kinesis.processor.IPreparedCheckpointer; import software.amazon.kinesis.processor.IRecordProcessorCheckpointer; import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; @@ -49,7 +49,7 @@ public class RecordProcessorCheckpointer implements IRecordProcessorCheckpointer @NonNull private final ShardInfo shardInfo; @NonNull - private final ICheckpoint checkpoint; + private final Checkpointer checkpoint; @NonNull private final IMetricsFactory metricsFactory; diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/CoordinatorFactory.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/CoordinatorFactory.java index 62be0b19..dfbbeb58 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/CoordinatorFactory.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/CoordinatorFactory.java @@ -17,10 +17,10 @@ package software.amazon.kinesis.coordinator; import java.util.concurrent.ExecutorService; -import software.amazon.kinesis.checkpoint.Checkpoint; +import software.amazon.kinesis.checkpoint.RecordProcessorCheckpointer; import software.amazon.kinesis.leases.ShardInfo; import software.amazon.kinesis.metrics.IMetricsFactory; -import software.amazon.kinesis.processor.ICheckpoint; +import software.amazon.kinesis.processor.Checkpointer; /** * @@ -32,6 +32,6 @@ public interface CoordinatorFactory { WorkerStateChangeListener createWorkerStateChangeListener(); - RecordProcessorCheckpointer createRecordProcessorCheckpointer(ShardInfo shardInfo, ICheckpoint checkpoint, + RecordProcessorCheckpointer createRecordProcessorCheckpointer(ShardInfo shardInfo, Checkpointer checkpoint, IMetricsFactory metricsFactory); } diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/KinesisClientLibConfiguration.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/KinesisClientLibConfiguration.java index b8fb8bdc..6d130b1d 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/KinesisClientLibConfiguration.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/KinesisClientLibConfiguration.java @@ -28,6 +28,7 @@ import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionIn import com.google.common.collect.ImmutableSet; import lombok.Getter; +import software.amazon.kinesis.checkpoint.RecordProcessorCheckpointer; import software.amazon.kinesis.leases.NoOpShardPrioritization; import software.amazon.kinesis.leases.ShardPrioritization; import software.amazon.kinesis.lifecycle.ProcessRecordsInput; @@ -36,7 +37,7 @@ import software.amazon.kinesis.lifecycle.ShardConsumer; import software.amazon.kinesis.metrics.IMetricsScope; import software.amazon.kinesis.metrics.MetricsHelper; import software.amazon.kinesis.metrics.MetricsLevel; -import software.amazon.kinesis.processor.IRecordProcessor; +import software.amazon.kinesis.processor.RecordProcessor; import software.amazon.kinesis.retrieval.DataFetchingStrategy; import software.amazon.kinesis.retrieval.RecordsFetcherFactory; import software.amazon.kinesis.retrieval.SimpleRecordsFetcherFactory; @@ -1005,7 +1006,7 @@ public class KinesisClientLibConfiguration { *

* This value is only used when no records are returned; if records are returned, the {@link ProcessTask} will * immediately retrieve the next set of records after the call to - * {@link IRecordProcessor#processRecords(ProcessRecordsInput)} + * {@link RecordProcessor#processRecords(ProcessRecordsInput)} * has returned. Setting this value to high may result in the KCL being unable to catch up. If you are changing this * value it's recommended that you enable {@link #withCallProcessRecordsEvenForEmptyRecordList(boolean)}, and * monitor how far behind the records retrieved are by inspecting diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/Scheduler.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/Scheduler.java index 835f0e76..cbe3a336 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/Scheduler.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/Scheduler.java @@ -59,8 +59,8 @@ import software.amazon.kinesis.metrics.IMetricsFactory; import software.amazon.kinesis.metrics.MetricsCollectingTaskDecorator; import software.amazon.kinesis.metrics.MetricsConfig; import software.amazon.kinesis.metrics.MetricsLevel; -import software.amazon.kinesis.processor.ICheckpoint; -import software.amazon.kinesis.processor.IShutdownNotificationAware; +import software.amazon.kinesis.processor.Checkpointer; +import software.amazon.kinesis.processor.ShutdownNotificationAware; import software.amazon.kinesis.processor.ProcessorConfig; import software.amazon.kinesis.processor.ProcessorFactory; import software.amazon.kinesis.retrieval.RetrievalConfig; @@ -84,7 +84,7 @@ public class Scheduler implements Runnable { private final RetrievalConfig retrievalConfig; private final String applicationName; - private final ICheckpoint checkpoint; + private final Checkpointer checkpoint; private final long idleTimeInMilliseconds; // Backoff time when polling to check if application has finished processing // parent shards @@ -320,7 +320,7 @@ public class Scheduler implements Runnable { /** * Requests a graceful shutdown of the worker, notifying record processors, that implement - * {@link IShutdownNotificationAware}, of the impending shutdown. This gives the record processor a final chance to + * {@link ShutdownNotificationAware}, of the impending shutdown. This gives the record processor a final chance to * checkpoint. * * This will only create a single shutdown future. Additional attempts to start a graceful shutdown will return the diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/SchedulerCoordinatorFactory.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/SchedulerCoordinatorFactory.java index 7cec4243..ec4ee7ac 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/SchedulerCoordinatorFactory.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/SchedulerCoordinatorFactory.java @@ -25,10 +25,10 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import lombok.Data; import lombok.NonNull; -import software.amazon.kinesis.checkpoint.Checkpoint; +import software.amazon.kinesis.checkpoint.RecordProcessorCheckpointer; import software.amazon.kinesis.leases.ShardInfo; import software.amazon.kinesis.metrics.IMetricsFactory; -import software.amazon.kinesis.processor.ICheckpoint; +import software.amazon.kinesis.processor.Checkpointer; /** * @@ -61,7 +61,7 @@ public class SchedulerCoordinatorFactory implements CoordinatorFactory { @Override public RecordProcessorCheckpointer createRecordProcessorCheckpointer(@NonNull final ShardInfo shardInfo, - @NonNull final ICheckpoint checkpoint, + @NonNull final Checkpointer checkpoint, @NonNull final IMetricsFactory metricsFactory) { return new RecordProcessorCheckpointer(shardInfo, checkpoint, metricsFactory); } diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/DynamoDBLeaseRenewer.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/DynamoDBLeaseRenewer.java new file mode 100644 index 00000000..330ec30e --- /dev/null +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/DynamoDBLeaseRenewer.java @@ -0,0 +1,410 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package software.amazon.kinesis.leases; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentNavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import com.amazonaws.services.cloudwatch.model.StandardUnit; +import software.amazon.kinesis.leases.exceptions.DependencyException; +import software.amazon.kinesis.leases.exceptions.InvalidStateException; +import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException; +import software.amazon.kinesis.metrics.MetricsHelper; +import software.amazon.kinesis.metrics.ThreadSafeMetricsDelegatingScope; +import software.amazon.kinesis.metrics.IMetricsScope; +import software.amazon.kinesis.metrics.MetricsLevel; + +import lombok.extern.slf4j.Slf4j; + +/** + * An implementation of ILeaseRenewer that uses DynamoDB via LeaseManager. + */ +@Slf4j +public class LeaseRenewer implements ILeaseRenewer { + private static final int RENEWAL_RETRIES = 2; + + private final ILeaseManager leaseManager; + private final ConcurrentNavigableMap ownedLeases = new ConcurrentSkipListMap(); + private final String workerIdentifier; + private final long leaseDurationNanos; + private final ExecutorService executorService; + + /** + * Constructor. + * + * @param leaseManager LeaseManager to use + * @param workerIdentifier identifier of this worker + * @param leaseDurationMillis duration of a lease in milliseconds + * @param executorService ExecutorService to use for renewing leases in parallel + */ + public LeaseRenewer(ILeaseManager leaseManager, String workerIdentifier, long leaseDurationMillis, + ExecutorService executorService) { + this.leaseManager = leaseManager; + this.workerIdentifier = workerIdentifier; + this.leaseDurationNanos = TimeUnit.MILLISECONDS.toNanos(leaseDurationMillis); + this.executorService = executorService; + } + + /** + * {@inheritDoc} + */ + @Override + public void renewLeases() throws DependencyException, InvalidStateException { + if (log.isDebugEnabled()) { + // Due to the eventually consistent nature of ConcurrentNavigableMap iterators, this log entry may become + // inaccurate during iteration. + log.debug("Worker {} holding %d leases: {}", + workerIdentifier, + ownedLeases.size(), + ownedLeases); + } + + /* + * Lease renewals are done in parallel so many leases can be renewed for short lease fail over time + * configuration. In this case, metrics scope is also shared across different threads, so scope must be thread + * safe. + */ + IMetricsScope renewLeaseTaskMetricsScope = new ThreadSafeMetricsDelegatingScope( + MetricsHelper.getMetricsScope()); + + /* + * We iterate in descending order here so that the synchronized(lease) inside renewLease doesn't "lead" calls + * to getCurrentlyHeldLeases. They'll still cross paths, but they won't interleave their executions. + */ + int lostLeases = 0; + List> renewLeaseTasks = new ArrayList>(); + for (T lease : ownedLeases.descendingMap().values()) { + renewLeaseTasks.add(executorService.submit(new RenewLeaseTask(lease, renewLeaseTaskMetricsScope))); + } + int leasesInUnknownState = 0; + Exception lastException = null; + for (Future renewLeaseTask : renewLeaseTasks) { + try { + if (!renewLeaseTask.get()) { + lostLeases++; + } + } catch (InterruptedException e) { + log.info("Interrupted while waiting for a lease to renew."); + leasesInUnknownState += 1; + Thread.currentThread().interrupt(); + } catch (ExecutionException e) { + log.error("Encountered an exception while renewing a lease.", e.getCause()); + leasesInUnknownState += 1; + lastException = e; + } + } + + renewLeaseTaskMetricsScope.addData( + "LostLeases", lostLeases, StandardUnit.Count, MetricsLevel.SUMMARY); + renewLeaseTaskMetricsScope.addData( + "CurrentLeases", ownedLeases.size(), StandardUnit.Count, MetricsLevel.SUMMARY); + if (leasesInUnknownState > 0) { + throw new DependencyException(String.format("Encountered an exception while renewing leases. " + + "The number of leases which might not have been renewed is %d", + leasesInUnknownState), + lastException); + } + } + + private class RenewLeaseTask implements Callable { + + private final T lease; + private final IMetricsScope metricsScope; + + public RenewLeaseTask(T lease, IMetricsScope metricsScope) { + this.lease = lease; + this.metricsScope = metricsScope; + } + + @Override + public Boolean call() throws Exception { + MetricsHelper.setMetricsScope(metricsScope); + try { + return renewLease(lease); + } finally { + MetricsHelper.unsetMetricsScope(); + } + } + } + + private boolean renewLease(T lease) throws DependencyException, InvalidStateException { + return renewLease(lease, false); + } + + private boolean renewLease(T lease, boolean renewEvenIfExpired) throws DependencyException, InvalidStateException { + String leaseKey = lease.getLeaseKey(); + + boolean success = false; + boolean renewedLease = false; + long startTime = System.currentTimeMillis(); + try { + for (int i = 1; i <= RENEWAL_RETRIES; i++) { + try { + synchronized (lease) { + // Don't renew expired lease during regular renewals. getCopyOfHeldLease may have returned null + // triggering the application processing to treat this as a lost lease (fail checkpoint with + // ShutdownException). + boolean isLeaseExpired = lease.isExpired(leaseDurationNanos, System.nanoTime()); + if (renewEvenIfExpired || !isLeaseExpired) { + renewedLease = leaseManager.renewLease(lease); + } + if (renewedLease) { + lease.setLastCounterIncrementNanos(System.nanoTime()); + } + } + + if (renewedLease) { + if (log.isDebugEnabled()) { + log.debug("Worker {} successfully renewed lease with key {}", + workerIdentifier, + leaseKey); + } + } else { + log.info("Worker {} lost lease with key {}", workerIdentifier, leaseKey); + ownedLeases.remove(leaseKey); + } + + success = true; + break; + } catch (ProvisionedThroughputException e) { + log.info("Worker {} could not renew lease with key {} on try {} out of {} due to capacity", + workerIdentifier, + leaseKey, + i, + RENEWAL_RETRIES); + } + } + } finally { + MetricsHelper.addSuccessAndLatency("RenewLease", startTime, success, MetricsLevel.DETAILED); + } + + return renewedLease; + } + + /** + * {@inheritDoc} + */ + @Override + public Map getCurrentlyHeldLeases() { + Map result = new HashMap(); + long now = System.nanoTime(); + + for (String leaseKey : ownedLeases.keySet()) { + T copy = getCopyOfHeldLease(leaseKey, now); + if (copy != null) { + result.put(copy.getLeaseKey(), copy); + } + } + + return result; + } + + /** + * {@inheritDoc} + */ + @Override + public T getCurrentlyHeldLease(String leaseKey) { + return getCopyOfHeldLease(leaseKey, System.nanoTime()); + } + + /** + * Internal method to return a lease with a specific lease key only if we currently hold it. + * + * @param leaseKey key of lease to return + * @param now current timestamp for old-ness checking + * @return non-authoritative copy of the held lease, or null if we don't currently hold it + */ + private T getCopyOfHeldLease(String leaseKey, long now) { + T authoritativeLease = ownedLeases.get(leaseKey); + if (authoritativeLease == null) { + return null; + } else { + T copy = null; + synchronized (authoritativeLease) { + copy = authoritativeLease.copy(); + } + + if (copy.isExpired(leaseDurationNanos, now)) { + log.info("getCurrentlyHeldLease not returning lease with key {} because it is expired", + copy.getLeaseKey()); + return null; + } else { + return copy; + } + } + } + + /** + * {@inheritDoc} + */ + @Override + public boolean updateLease(T lease, UUID concurrencyToken) + throws DependencyException, InvalidStateException, ProvisionedThroughputException { + verifyNotNull(lease, "lease cannot be null"); + verifyNotNull(lease.getLeaseKey(), "leaseKey cannot be null"); + verifyNotNull(concurrencyToken, "concurrencyToken cannot be null"); + + String leaseKey = lease.getLeaseKey(); + T authoritativeLease = ownedLeases.get(leaseKey); + + if (authoritativeLease == null) { + log.info("Worker {} could not update lease with key {} because it does not hold it", + workerIdentifier, + leaseKey); + return false; + } + + /* + * If the passed-in concurrency token doesn't match the concurrency token of the authoritative lease, it means + * the lease was lost and regained between when the caller acquired his concurrency token and when the caller + * called update. + */ + if (!authoritativeLease.getConcurrencyToken().equals(concurrencyToken)) { + log.info("Worker {} refusing to update lease with key {} because" + + " concurrency tokens don't match", workerIdentifier, leaseKey); + return false; + } + + long startTime = System.currentTimeMillis(); + boolean success = false; + try { + synchronized (authoritativeLease) { + authoritativeLease.update(lease); + boolean updatedLease = leaseManager.updateLease(authoritativeLease); + if (updatedLease) { + // Updates increment the counter + authoritativeLease.setLastCounterIncrementNanos(System.nanoTime()); + } else { + /* + * If updateLease returns false, it means someone took the lease from us. Remove the lease + * from our set of owned leases pro-actively rather than waiting for a run of renewLeases(). + */ + log.info("Worker {} lost lease with key {} - discovered during update", + workerIdentifier, + leaseKey); + + /* + * Remove only if the value currently in the map is the same as the authoritative lease. We're + * guarding against a pause after the concurrency token check above. It plays out like so: + * + * 1) Concurrency token check passes + * 2) Pause. Lose lease, re-acquire lease. This requires at least one lease counter update. + * 3) Unpause. leaseManager.updateLease fails conditional write due to counter updates, returns + * false. + * 4) ownedLeases.remove(key, value) doesn't do anything because authoritativeLease does not + * .equals() the re-acquired version in the map on the basis of lease counter. This is what we want. + * If we just used ownedLease.remove(key), we would have pro-actively removed a lease incorrectly. + * + * Note that there is a subtlety here - Lease.equals() deliberately does not check the concurrency + * token, but it does check the lease counter, so this scheme works. + */ + ownedLeases.remove(leaseKey, authoritativeLease); + } + + success = true; + return updatedLease; + } + } finally { + MetricsHelper.addSuccessAndLatency("UpdateLease", startTime, success, MetricsLevel.DETAILED); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void addLeasesToRenew(Collection newLeases) { + verifyNotNull(newLeases, "newLeases cannot be null"); + + for (T lease : newLeases) { + if (lease.getLastCounterIncrementNanos() == null) { + log.info("addLeasesToRenew ignoring lease with key {} because it does not have lastRenewalNanos set", + lease.getLeaseKey()); + continue; + } + + T authoritativeLease = lease.copy(); + + /* + * Assign a concurrency token when we add this to the set of currently owned leases. This ensures that + * every time we acquire a lease, it gets a new concurrency token. + */ + authoritativeLease.setConcurrencyToken(UUID.randomUUID()); + ownedLeases.put(authoritativeLease.getLeaseKey(), authoritativeLease); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void clearCurrentlyHeldLeases() { + ownedLeases.clear(); + } + + /** + * {@inheritDoc} + * @param lease the lease to drop. + */ + @Override + public void dropLease(T lease) { + ownedLeases.remove(lease.getLeaseKey()); + } + + /** + * {@inheritDoc} + */ + @Override + public void initialize() throws DependencyException, InvalidStateException, ProvisionedThroughputException { + Collection leases = leaseManager.listLeases(); + List myLeases = new LinkedList(); + boolean renewEvenIfExpired = true; + + for (T lease : leases) { + if (workerIdentifier.equals(lease.getLeaseOwner())) { + log.info(" Worker {} found lease {}", workerIdentifier, lease); + // Okay to renew even if lease is expired, because we start with an empty list and we add the lease to + // our list only after a successful renew. So we don't need to worry about the edge case where we could + // continue renewing a lease after signaling a lease loss to the application. + if (renewLease(lease, renewEvenIfExpired)) { + myLeases.add(lease); + } + } else { + log.debug("Worker {} ignoring lease {} ", workerIdentifier, lease); + } + } + + addLeasesToRenew(myLeases); + } + + private void verifyNotNull(Object object, String message) { + if (object == null) { + throw new IllegalArgumentException(message); + } + } + +} diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/DynamoDBLeaseTaker.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/DynamoDBLeaseTaker.java new file mode 100644 index 00000000..16a07983 --- /dev/null +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/DynamoDBLeaseTaker.java @@ -0,0 +1,535 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package software.amazon.kinesis.leases; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; + +import com.amazonaws.services.cloudwatch.model.StandardUnit; +import software.amazon.kinesis.leases.exceptions.DependencyException; +import software.amazon.kinesis.leases.exceptions.InvalidStateException; +import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException; +import software.amazon.kinesis.metrics.MetricsHelper; +import software.amazon.kinesis.metrics.IMetricsScope; +import software.amazon.kinesis.metrics.MetricsLevel; + +import lombok.extern.slf4j.Slf4j; + +/** + * An implementation of ILeaseTaker that uses DynamoDB via LeaseManager. + */ +@Slf4j +public class DynamoDBLeaseTaker implements LeaseTaker { + private static final int TAKE_RETRIES = 3; + private static final int SCAN_RETRIES = 1; + + // See note on takeLeases(Callable) for why we have this callable. + private static final Callable SYSTEM_CLOCK_CALLABLE = new Callable() { + + @Override + public Long call() { + return System.nanoTime(); + } + }; + + private final ILeaseManager leaseManager; + private final String workerIdentifier; + private final Map allLeases = new HashMap(); + private final long leaseDurationNanos; + private int maxLeasesForWorker = Integer.MAX_VALUE; + private int maxLeasesToStealAtOneTime = 1; + + private long lastScanTimeNanos = 0L; + + public DynamoDBLeaseTaker(ILeaseManager leaseManager, String workerIdentifier, long leaseDurationMillis) { + this.leaseManager = leaseManager; + this.workerIdentifier = workerIdentifier; + this.leaseDurationNanos = TimeUnit.MILLISECONDS.toNanos(leaseDurationMillis); + } + + /** + * Worker will not acquire more than the specified max number of leases even if there are more + * shards that need to be processed. This can be used in scenarios where a worker is resource constrained or + * to prevent lease thrashing when small number of workers pick up all leases for small amount of time during + * deployment. + * Note that setting a low value may cause data loss (e.g. if there aren't enough Workers to make progress on all + * shards). When setting the value for this property, one must ensure enough workers are present to process + * shards and should consider future resharding, child shards that may be blocked on parent shards, some workers + * becoming unhealthy, etc. + * + * @param maxLeasesForWorker Max leases this Worker can handle at a time + * @return LeaseTaker + */ + public DynamoDBLeaseTaker withMaxLeasesForWorker(int maxLeasesForWorker) { + if (maxLeasesForWorker <= 0) { + throw new IllegalArgumentException("maxLeasesForWorker should be >= 1"); + } + this.maxLeasesForWorker = maxLeasesForWorker; + return this; + } + + /** + * Max leases to steal from a more loaded Worker at one time (for load balancing). + * Setting this to a higher number can allow for faster load convergence (e.g. during deployments, cold starts), + * but can cause higher churn in the system. + * + * @param maxLeasesToStealAtOneTime Steal up to this many leases at one time (for load balancing) + * @return LeaseTaker + */ + public DynamoDBLeaseTaker withMaxLeasesToStealAtOneTime(int maxLeasesToStealAtOneTime) { + if (maxLeasesToStealAtOneTime <= 0) { + throw new IllegalArgumentException("maxLeasesToStealAtOneTime should be >= 1"); + } + this.maxLeasesToStealAtOneTime = maxLeasesToStealAtOneTime; + return this; + } + + /** + * {@inheritDoc} + */ + @Override + public Map takeLeases() throws DependencyException, InvalidStateException { + return takeLeases(SYSTEM_CLOCK_CALLABLE); + } + + /** + * Internal implementation of takeLeases. Takes a callable that can provide the time to enable test cases without + * Thread.sleep. Takes a callable instead of a raw time value because the time needs to be computed as-of + * immediately after the scan. + * + * @param timeProvider Callable that will supply the time + * + * @return map of lease key to taken lease + * + * @throws DependencyException + * @throws InvalidStateException + */ + synchronized Map takeLeases(Callable timeProvider) + throws DependencyException, InvalidStateException { + // Key is leaseKey + Map takenLeases = new HashMap(); + + long startTime = System.currentTimeMillis(); + boolean success = false; + + ProvisionedThroughputException lastException = null; + + try { + for (int i = 1; i <= SCAN_RETRIES; i++) { + try { + updateAllLeases(timeProvider); + success = true; + } catch (ProvisionedThroughputException e) { + log.info("Worker {} could not find expired leases on try {} out of {}", + workerIdentifier, + i, + TAKE_RETRIES); + lastException = e; + } + } + } finally { + MetricsHelper.addSuccessAndLatency("ListLeases", startTime, success, MetricsLevel.DETAILED); + } + + if (lastException != null) { + log.error("Worker {} could not scan leases table, aborting takeLeases. Exception caught by last retry:", + workerIdentifier, + lastException); + return takenLeases; + } + + List expiredLeases = getExpiredLeases(); + + Set leasesToTake = computeLeasesToTake(expiredLeases); + Set untakenLeaseKeys = new HashSet(); + + for (T lease : leasesToTake) { + String leaseKey = lease.getLeaseKey(); + + startTime = System.currentTimeMillis(); + success = false; + try { + for (int i = 1; i <= TAKE_RETRIES; i++) { + try { + if (leaseManager.takeLease(lease, workerIdentifier)) { + lease.setLastCounterIncrementNanos(System.nanoTime()); + takenLeases.put(leaseKey, lease); + } else { + untakenLeaseKeys.add(leaseKey); + } + + success = true; + break; + } catch (ProvisionedThroughputException e) { + log.info("Could not take lease with key {} for worker {} on try {} out of {} due to capacity", + leaseKey, + workerIdentifier, + i, + TAKE_RETRIES); + } + } + } finally { + MetricsHelper.addSuccessAndLatency("TakeLease", startTime, success, MetricsLevel.DETAILED); + } + } + + if (takenLeases.size() > 0) { + log.info("Worker {} successfully took {} leases: {}", + workerIdentifier, + takenLeases.size(), + stringJoin(takenLeases.keySet(), ", ")); + } + + if (untakenLeaseKeys.size() > 0) { + log.info("Worker {} failed to take {} leases: {}", + workerIdentifier, + untakenLeaseKeys.size(), + stringJoin(untakenLeaseKeys, ", ")); + } + + MetricsHelper.getMetricsScope().addData( + "TakenLeases", takenLeases.size(), StandardUnit.Count, MetricsLevel.SUMMARY); + + return takenLeases; + } + + /** Package access for testing purposes. + * + * @param strings + * @param delimiter + * @return Joined string. + */ + static String stringJoin(Collection strings, String delimiter) { + StringBuilder builder = new StringBuilder(); + boolean needDelimiter = false; + for (String string : strings) { + if (needDelimiter) { + builder.append(delimiter); + } + builder.append(string); + needDelimiter = true; + } + + return builder.toString(); + } + + /** + * Scan all leases and update lastRenewalTime. Add new leases and delete old leases. + * + * @param timeProvider callable that supplies the current time + * + * @return list of expired leases, possibly empty, never null. + * + * @throws ProvisionedThroughputException if listLeases fails due to lack of provisioned throughput + * @throws InvalidStateException if the lease table does not exist + * @throws DependencyException if listLeases fails in an unexpected way + */ + private void updateAllLeases(Callable timeProvider) + throws DependencyException, InvalidStateException, ProvisionedThroughputException { + List freshList = leaseManager.listLeases(); + try { + lastScanTimeNanos = timeProvider.call(); + } catch (Exception e) { + throw new DependencyException("Exception caught from timeProvider", e); + } + + // This set will hold the lease keys not updated by the previous listLeases call. + Set notUpdated = new HashSet(allLeases.keySet()); + + // Iterate over all leases, finding ones to try to acquire that haven't changed since the last iteration + for (T lease : freshList) { + String leaseKey = lease.getLeaseKey(); + + T oldLease = allLeases.get(leaseKey); + allLeases.put(leaseKey, lease); + notUpdated.remove(leaseKey); + + if (oldLease != null) { + // If we've seen this lease before... + if (oldLease.getLeaseCounter().equals(lease.getLeaseCounter())) { + // ...and the counter hasn't changed, propagate the lastRenewalNanos time from the old lease + lease.setLastCounterIncrementNanos(oldLease.getLastCounterIncrementNanos()); + } else { + // ...and the counter has changed, set lastRenewalNanos to the time of the scan. + lease.setLastCounterIncrementNanos(lastScanTimeNanos); + } + } else { + if (lease.getLeaseOwner() == null) { + // if this new lease is unowned, it's never been renewed. + lease.setLastCounterIncrementNanos(0L); + + if (log.isDebugEnabled()) { + log.debug("Treating new lease with key {} as never renewed because it is new and unowned.", + leaseKey); + } + } else { + // if this new lease is owned, treat it as renewed as of the scan + lease.setLastCounterIncrementNanos(lastScanTimeNanos); + if (log.isDebugEnabled()) { + log.debug("Treating new lease with key {} as recently renewed because it is new and owned.", + leaseKey); + } + } + } + } + + // Remove dead leases from allLeases + for (String key : notUpdated) { + allLeases.remove(key); + } + } + + /** + * @return list of leases that were expired as of our last scan. + */ + private List getExpiredLeases() { + List expiredLeases = new ArrayList(); + + for (T lease : allLeases.values()) { + if (lease.isExpired(leaseDurationNanos, lastScanTimeNanos)) { + expiredLeases.add(lease); + } + } + + return expiredLeases; + } + + /** + * Compute the number of leases I should try to take based on the state of the system. + * + * @param allLeases map of shardId to lease containing all leases + * @param expiredLeases list of leases we determined to be expired + * @return set of leases to take. + */ + private Set computeLeasesToTake(List expiredLeases) { + Map leaseCounts = computeLeaseCounts(expiredLeases); + Set leasesToTake = new HashSet(); + IMetricsScope metrics = MetricsHelper.getMetricsScope(); + + int numLeases = allLeases.size(); + int numWorkers = leaseCounts.size(); + + if (numLeases == 0) { + // If there are no leases, I shouldn't try to take any. + return leasesToTake; + } + + int target; + if (numWorkers >= numLeases) { + // If we have n leases and n or more workers, each worker can have up to 1 lease, including myself. + target = 1; + } else { + /* + * numWorkers must be < numLeases. + * + * Our target for each worker is numLeases / numWorkers (+1 if numWorkers doesn't evenly divide numLeases) + */ + target = numLeases / numWorkers + (numLeases % numWorkers == 0 ? 0 : 1); + + // Spill over is the number of leases this worker should have claimed, but did not because it would + // exceed the max allowed for this worker. + int leaseSpillover = Math.max(0, target - maxLeasesForWorker); + if (target > maxLeasesForWorker) { + log.warn("Worker {} target is {} leases and maxLeasesForWorker is {}." + + " Resetting target to {}, lease spillover is {}. " + + " Note that some shards may not be processed if no other workers are able to pick them up.", + workerIdentifier, + target, + maxLeasesForWorker, + maxLeasesForWorker, + leaseSpillover); + target = maxLeasesForWorker; + } + metrics.addData("LeaseSpillover", leaseSpillover, StandardUnit.Count, MetricsLevel.SUMMARY); + } + + int myCount = leaseCounts.get(workerIdentifier); + int numLeasesToReachTarget = target - myCount; + + if (numLeasesToReachTarget <= 0) { + // If we don't need anything, return the empty set. + return leasesToTake; + } + + // Shuffle expiredLeases so workers don't all try to contend for the same leases. + Collections.shuffle(expiredLeases); + + int originalExpiredLeasesSize = expiredLeases.size(); + if (expiredLeases.size() > 0) { + // If we have expired leases, get up to leases from expiredLeases + for (; numLeasesToReachTarget > 0 && expiredLeases.size() > 0; numLeasesToReachTarget--) { + leasesToTake.add(expiredLeases.remove(0)); + } + } else { + // If there are no expired leases and we need a lease, consider stealing. + List leasesToSteal = chooseLeasesToSteal(leaseCounts, numLeasesToReachTarget, target); + for (T leaseToSteal : leasesToSteal) { + log.info("Worker {} needed {} leases but none were expired, so it will steal lease {} from {}", + workerIdentifier, + numLeasesToReachTarget, + leaseToSteal.getLeaseKey(), + leaseToSteal.getLeaseOwner()); + leasesToTake.add(leaseToSteal); + } + } + + if (!leasesToTake.isEmpty()) { + log.info("Worker {} saw {} total leases, {} available leases, {} " + + "workers. Target is {} leases, I have {} leases, I will take {} leases", + workerIdentifier, + numLeases, + originalExpiredLeasesSize, + numWorkers, + target, + myCount, + leasesToTake.size()); + } + + metrics.addData("TotalLeases", numLeases, StandardUnit.Count, MetricsLevel.DETAILED); + metrics.addData("ExpiredLeases", originalExpiredLeasesSize, StandardUnit.Count, MetricsLevel.SUMMARY); + metrics.addData("NumWorkers", numWorkers, StandardUnit.Count, MetricsLevel.SUMMARY); + metrics.addData("NeededLeases", numLeasesToReachTarget, StandardUnit.Count, MetricsLevel.DETAILED); + metrics.addData("LeasesToTake", leasesToTake.size(), StandardUnit.Count, MetricsLevel.DETAILED); + + return leasesToTake; + } + + /** + * Choose leases to steal by randomly selecting one or more (up to max) from the most loaded worker. + * Stealing rules: + * + * Steal up to maxLeasesToStealAtOneTime leases from the most loaded worker if + * a) he has > target leases and I need >= 1 leases : steal min(leases needed, maxLeasesToStealAtOneTime) + * b) he has == target leases and I need > 1 leases : steal 1 + * + * @param leaseCounts map of workerIdentifier to lease count + * @param needed # of leases needed to reach the target leases for the worker + * @param target target # of leases per worker + * @return Leases to steal, or empty list if we should not steal + */ + private List chooseLeasesToSteal(Map leaseCounts, int needed, int target) { + List leasesToSteal = new ArrayList<>(); + + Entry mostLoadedWorker = null; + // Find the most loaded worker + for (Entry worker : leaseCounts.entrySet()) { + if (mostLoadedWorker == null || mostLoadedWorker.getValue() < worker.getValue()) { + mostLoadedWorker = worker; + } + } + + int numLeasesToSteal = 0; + if ((mostLoadedWorker.getValue() >= target) && (needed > 0)) { + int leasesOverTarget = mostLoadedWorker.getValue() - target; + numLeasesToSteal = Math.min(needed, leasesOverTarget); + // steal 1 if we need > 1 and max loaded worker has target leases. + if ((needed > 1) && (numLeasesToSteal == 0)) { + numLeasesToSteal = 1; + } + numLeasesToSteal = Math.min(numLeasesToSteal, maxLeasesToStealAtOneTime); + } + + if (numLeasesToSteal <= 0) { + if (log.isDebugEnabled()) { + log.debug(String.format("Worker %s not stealing from most loaded worker %s. He has %d," + + " target is %d, and I need %d", + workerIdentifier, + mostLoadedWorker.getKey(), + mostLoadedWorker.getValue(), + target, + needed)); + } + return leasesToSteal; + } else { + if (log.isDebugEnabled()) { + log.debug("Worker {} will attempt to steal {} leases from most loaded worker {}. " + + " He has {} leases, target is {}, I need {}, maxLeasesToSteatAtOneTime is {}.", + workerIdentifier, + numLeasesToSteal, + mostLoadedWorker.getKey(), + mostLoadedWorker.getValue(), + target, + needed, + maxLeasesToStealAtOneTime); + } + } + + String mostLoadedWorkerIdentifier = mostLoadedWorker.getKey(); + List candidates = new ArrayList(); + // Collect leases belonging to that worker + for (T lease : allLeases.values()) { + if (mostLoadedWorkerIdentifier.equals(lease.getLeaseOwner())) { + candidates.add(lease); + } + } + + // Return random ones + Collections.shuffle(candidates); + int toIndex = Math.min(candidates.size(), numLeasesToSteal); + leasesToSteal.addAll(candidates.subList(0, toIndex)); + + return leasesToSteal; + } + + /** + * Count leases by host. Always includes myself, but otherwise only includes hosts that are currently holding + * leases. + * + * @param expiredLeases list of leases that are currently expired + * @return map of workerIdentifier to lease count + */ + private Map computeLeaseCounts(List expiredLeases) { + Map leaseCounts = new HashMap(); + + // Compute the number of leases per worker by looking through allLeases and ignoring leases that have expired. + for (T lease : allLeases.values()) { + if (!expiredLeases.contains(lease)) { + String leaseOwner = lease.getLeaseOwner(); + Integer oldCount = leaseCounts.get(leaseOwner); + if (oldCount == null) { + leaseCounts.put(leaseOwner, 1); + } else { + leaseCounts.put(leaseOwner, oldCount + 1); + } + } + } + + // If I have no leases, I wasn't represented in leaseCounts. Let's fix that. + Integer myCount = leaseCounts.get(workerIdentifier); + if (myCount == null) { + myCount = 0; + leaseCounts.put(workerIdentifier, myCount); + } + + return leaseCounts; + } + + /** + * {@inheritDoc} + */ + @Override + public String getWorkerIdentifier() { + return workerIdentifier; + } +} diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ILeaseRenewer.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ILeaseRenewer.java deleted file mode 100644 index 8f681531..00000000 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ILeaseRenewer.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/asl/ - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ -package software.amazon.kinesis.leases; - -import java.util.Collection; -import java.util.Map; -import java.util.UUID; - -import software.amazon.kinesis.leases.exceptions.DependencyException; -import software.amazon.kinesis.leases.exceptions.InvalidStateException; -import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException; -import software.amazon.kinesis.leases.Lease; - -/** - * ILeaseRenewer objects are used by LeaseCoordinator to renew leases held by the LeaseCoordinator. Each - * LeaseCoordinator instance corresponds to one worker, and uses exactly one ILeaseRenewer to manage lease renewal for - * that worker. - */ -public interface ILeaseRenewer { - - /** - * Bootstrap initial set of leases from the LeaseManager (e.g. upon process restart, pick up leases we own) - * @throws DependencyException on unexpected DynamoDB failures - * @throws InvalidStateException if lease table doesn't exist - * @throws ProvisionedThroughputException if DynamoDB reads fail due to insufficient capacity - */ - public void initialize() throws DependencyException, InvalidStateException, ProvisionedThroughputException; - - /** - * Attempt to renew all currently held leases. - * - * @throws DependencyException on unexpected DynamoDB failures - * @throws InvalidStateException if lease table does not exist - */ - public void renewLeases() throws DependencyException, InvalidStateException; - - /** - * @return currently held leases. Key is shardId, value is corresponding Lease object. A lease is currently held if - * we successfully renewed it on the last run of renewLeases(). Lease objects returned are deep copies - - * their lease counters will not tick. - */ - public Map getCurrentlyHeldLeases(); - - /** - * @param leaseKey key of the lease to retrieve - * - * @return a deep copy of a currently held lease, or null if we don't hold the lease - */ - public T getCurrentlyHeldLease(String leaseKey); - - /** - * Adds leases to this LeaseRenewer's set of currently held leases. Leases must have lastRenewalNanos set to the - * last time the lease counter was incremented before being passed to this method. - * - * @param newLeases new leases. - */ - public void addLeasesToRenew(Collection newLeases); - - /** - * Clears this LeaseRenewer's set of currently held leases. - */ - public void clearCurrentlyHeldLeases(); - - /** - * Stops the lease renewer from continunig to maintain the given lease. - * - * @param lease the lease to drop. - */ - void dropLease(T lease); - - /** - * Update application-specific fields in a currently held lease. Cannot be used to update internal fields such as - * leaseCounter, leaseOwner, etc. Fails if we do not hold the lease, or if the concurrency token does not match - * the concurrency token on the internal authoritative copy of the lease (ie, if we lost and re-acquired the lease). - * - * @param lease lease object containing updated data - * @param concurrencyToken obtained by calling Lease.getConcurrencyToken for a currently held lease - * - * @return true if update succeeds, false otherwise - * - * @throws InvalidStateException if lease table does not exist - * @throws ProvisionedThroughputException if DynamoDB update fails due to lack of capacity - * @throws DependencyException if DynamoDB update fails in an unexpected way - */ - boolean updateLease(T lease, UUID concurrencyToken) - throws DependencyException, InvalidStateException, ProvisionedThroughputException; - -} diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ILeaseTaker.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ILeaseTaker.java deleted file mode 100644 index 54b5546e..00000000 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ILeaseTaker.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/asl/ - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ -package software.amazon.kinesis.leases; - -import java.util.Map; - -import software.amazon.kinesis.leases.exceptions.DependencyException; -import software.amazon.kinesis.leases.exceptions.InvalidStateException; -import software.amazon.kinesis.leases.Lease; - -/** - * ILeaseTaker is used by LeaseCoordinator to take new leases, or leases that other workers fail to renew. Each - * LeaseCoordinator instance corresponds to one worker and uses exactly one ILeaseTaker to take leases for that worker. - */ -public interface ILeaseTaker { - - /** - * Compute the set of leases available to be taken and attempt to take them. Lease taking rules are: - * - * 1) If a lease's counter hasn't changed in long enough, try to take it. - * 2) If we see a lease we've never seen before, take it only if owner == null. If it's owned, odds are the owner is - * holding it. We can't tell until we see it more than once. - * 3) For load balancing purposes, you may violate rules 1 and 2 for EXACTLY ONE lease per call of takeLeases(). - * - * @return map of shardId to Lease object for leases we just successfully took. - * - * @throws DependencyException on unexpected DynamoDB failures - * @throws InvalidStateException if lease table does not exist - */ - public abstract Map takeLeases() throws DependencyException, InvalidStateException; - - /** - * @return workerIdentifier for this LeaseTaker - */ - public abstract String getWorkerIdentifier(); - -} diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/KinesisClientLibLeaseCoordinator.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/KinesisClientLibLeaseCoordinator.java index 928cbc16..1a450a3c 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/KinesisClientLibLeaseCoordinator.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/KinesisClientLibLeaseCoordinator.java @@ -18,41 +18,28 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.Objects; import java.util.Set; -import java.util.UUID; -import com.amazonaws.services.kinesis.clientlibrary.exceptions.KinesisClientLibDependencyException; -import com.amazonaws.services.kinesis.clientlibrary.exceptions.KinesisClientLibException; -import com.amazonaws.services.kinesis.clientlibrary.exceptions.ShutdownException; -import com.amazonaws.services.kinesis.clientlibrary.exceptions.ThrottlingException; -import com.amazonaws.services.kinesis.clientlibrary.exceptions.internal.KinesisClientLibIOException; +import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; import lombok.experimental.Accessors; -import software.amazon.kinesis.processor.ICheckpoint; -import software.amazon.kinesis.checkpoint.Checkpoint; -import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; +import lombok.extern.slf4j.Slf4j; import software.amazon.kinesis.leases.exceptions.DependencyException; import software.amazon.kinesis.leases.exceptions.InvalidStateException; import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException; import software.amazon.kinesis.metrics.IMetricsFactory; -import lombok.extern.slf4j.Slf4j; - /** * This class is used to coordinate/manage leases owned by this worker process and to get/set checkpoints. */ @Slf4j -public class KinesisClientLibLeaseCoordinator extends LeaseCoordinator implements ICheckpoint { +public class KinesisClientLibLeaseCoordinator extends LeaseCoordinator { private static final long DEFAULT_INITIAL_LEASE_TABLE_READ_CAPACITY = 10L; private static final long DEFAULT_INITIAL_LEASE_TABLE_WRITE_CAPACITY = 10L; - /** - * Used to get information about leases for Kinesis shards (e.g. sync shards and leases, check on parent shard - * completion). - * - * @return LeaseManager - */ @Getter @Accessors(fluent = true) private final ILeaseManager leaseManager; @@ -60,53 +47,14 @@ public class KinesisClientLibLeaseCoordinator extends LeaseCoordinator leaseManager, - String workerIdentifier, - long leaseDurationMillis, - long epsilonMillis) { - super(leaseManager, workerIdentifier, leaseDurationMillis, epsilonMillis); - this.leaseManager = leaseManager; - } - - /** - * @param leaseManager Lease manager which provides CRUD lease operations. - * @param workerIdentifier Used to identify this worker process - * @param leaseDurationMillis Duration of a lease in milliseconds - * @param epsilonMillis Delta for timing operations (e.g. checking lease expiry) - * @param metricsFactory Metrics factory used to emit metrics - */ - public KinesisClientLibLeaseCoordinator(ILeaseManager leaseManager, - String workerIdentifier, - long leaseDurationMillis, - long epsilonMillis, - IMetricsFactory metricsFactory) { - super(leaseManager, workerIdentifier, leaseDurationMillis, epsilonMillis, metricsFactory); - this.leaseManager = leaseManager; - } - - /** - * @param leaseManager Lease manager which provides CRUD lease operations. - * @param workerIdentifier Used to identify this worker process - * @param leaseDurationMillis Duration of a lease in milliseconds - * @param epsilonMillis Delta for timing operations (e.g. checking lease expiry) - * @param maxLeasesForWorker Max leases this worker can handle at a time - * @param maxLeasesToStealAtOneTime Steal up to this many leases at a time (for load balancing) - * @param metricsFactory Metrics factory used to emit metrics - */ - public KinesisClientLibLeaseCoordinator(ILeaseManager leaseManager, - String workerIdentifier, - long leaseDurationMillis, - long epsilonMillis, - int maxLeasesForWorker, - int maxLeasesToStealAtOneTime, - int maxLeaseRenewerThreadCount, - IMetricsFactory metricsFactory) { + public KinesisClientLibLeaseCoordinator(final ILeaseManager leaseManager, + final String workerIdentifier, + final long leaseDurationMillis, + final long epsilonMillis, + final int maxLeasesForWorker, + final int maxLeasesToStealAtOneTime, + final int maxLeaseRenewerThreadCount, + final IMetricsFactory metricsFactory) { super(leaseManager, workerIdentifier, leaseDurationMillis, epsilonMillis, maxLeasesForWorker, maxLeasesToStealAtOneTime, maxLeaseRenewerThreadCount, metricsFactory); this.leaseManager = leaseManager; @@ -117,7 +65,7 @@ public class KinesisClientLibLeaseCoordinator extends LeaseCoordinator= 1"); } @@ -130,7 +78,7 @@ public class KinesisClientLibLeaseCoordinator extends LeaseCoordinator= 1"); } @@ -138,138 +86,6 @@ public class KinesisClientLibLeaseCoordinator extends LeaseCoordinator { .setNameFormat("LeaseRenewer-%04d").setDaemon(true).build(); private final ILeaseRenewer leaseRenewer; - private final ILeaseTaker leaseTaker; + private final LeaseTaker leaseTaker; private final long renewerIntervalMillis; private final long takerIntervalMillis; @@ -133,7 +133,7 @@ public class LeaseCoordinator { int maxLeaseRenewerThreadCount, IMetricsFactory metricsFactory) { this.leaseRenewalThreadpool = getLeaseRenewalExecutorService(maxLeaseRenewerThreadCount); - this.leaseTaker = new LeaseTaker(leaseManager, workerIdentifier, leaseDurationMillis) + this.leaseTaker = new DynamoDBLeaseTaker(leaseManager, workerIdentifier, leaseDurationMillis) .withMaxLeasesForWorker(maxLeasesForWorker) .withMaxLeasesToStealAtOneTime(maxLeasesToStealAtOneTime); this.leaseRenewer = new LeaseRenewer( diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseRenewer.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseRenewer.java index 330ec30e..8f681531 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseRenewer.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseRenewer.java @@ -14,397 +14,87 @@ */ package software.amazon.kinesis.leases; -import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; import java.util.Map; import java.util.UUID; -import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentNavigableMap; -import java.util.concurrent.ConcurrentSkipListMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import com.amazonaws.services.cloudwatch.model.StandardUnit; import software.amazon.kinesis.leases.exceptions.DependencyException; import software.amazon.kinesis.leases.exceptions.InvalidStateException; import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException; -import software.amazon.kinesis.metrics.MetricsHelper; -import software.amazon.kinesis.metrics.ThreadSafeMetricsDelegatingScope; -import software.amazon.kinesis.metrics.IMetricsScope; -import software.amazon.kinesis.metrics.MetricsLevel; - -import lombok.extern.slf4j.Slf4j; +import software.amazon.kinesis.leases.Lease; /** - * An implementation of ILeaseRenewer that uses DynamoDB via LeaseManager. + * ILeaseRenewer objects are used by LeaseCoordinator to renew leases held by the LeaseCoordinator. Each + * LeaseCoordinator instance corresponds to one worker, and uses exactly one ILeaseRenewer to manage lease renewal for + * that worker. */ -@Slf4j -public class LeaseRenewer implements ILeaseRenewer { - private static final int RENEWAL_RETRIES = 2; - - private final ILeaseManager leaseManager; - private final ConcurrentNavigableMap ownedLeases = new ConcurrentSkipListMap(); - private final String workerIdentifier; - private final long leaseDurationNanos; - private final ExecutorService executorService; +public interface ILeaseRenewer { + + /** + * Bootstrap initial set of leases from the LeaseManager (e.g. upon process restart, pick up leases we own) + * @throws DependencyException on unexpected DynamoDB failures + * @throws InvalidStateException if lease table doesn't exist + * @throws ProvisionedThroughputException if DynamoDB reads fail due to insufficient capacity + */ + public void initialize() throws DependencyException, InvalidStateException, ProvisionedThroughputException; /** - * Constructor. + * Attempt to renew all currently held leases. * - * @param leaseManager LeaseManager to use - * @param workerIdentifier identifier of this worker - * @param leaseDurationMillis duration of a lease in milliseconds - * @param executorService ExecutorService to use for renewing leases in parallel + * @throws DependencyException on unexpected DynamoDB failures + * @throws InvalidStateException if lease table does not exist */ - public LeaseRenewer(ILeaseManager leaseManager, String workerIdentifier, long leaseDurationMillis, - ExecutorService executorService) { - this.leaseManager = leaseManager; - this.workerIdentifier = workerIdentifier; - this.leaseDurationNanos = TimeUnit.MILLISECONDS.toNanos(leaseDurationMillis); - this.executorService = executorService; - } + public void renewLeases() throws DependencyException, InvalidStateException; /** - * {@inheritDoc} + * @return currently held leases. Key is shardId, value is corresponding Lease object. A lease is currently held if + * we successfully renewed it on the last run of renewLeases(). Lease objects returned are deep copies - + * their lease counters will not tick. */ - @Override - public void renewLeases() throws DependencyException, InvalidStateException { - if (log.isDebugEnabled()) { - // Due to the eventually consistent nature of ConcurrentNavigableMap iterators, this log entry may become - // inaccurate during iteration. - log.debug("Worker {} holding %d leases: {}", - workerIdentifier, - ownedLeases.size(), - ownedLeases); - } - - /* - * Lease renewals are done in parallel so many leases can be renewed for short lease fail over time - * configuration. In this case, metrics scope is also shared across different threads, so scope must be thread - * safe. - */ - IMetricsScope renewLeaseTaskMetricsScope = new ThreadSafeMetricsDelegatingScope( - MetricsHelper.getMetricsScope()); - - /* - * We iterate in descending order here so that the synchronized(lease) inside renewLease doesn't "lead" calls - * to getCurrentlyHeldLeases. They'll still cross paths, but they won't interleave their executions. - */ - int lostLeases = 0; - List> renewLeaseTasks = new ArrayList>(); - for (T lease : ownedLeases.descendingMap().values()) { - renewLeaseTasks.add(executorService.submit(new RenewLeaseTask(lease, renewLeaseTaskMetricsScope))); - } - int leasesInUnknownState = 0; - Exception lastException = null; - for (Future renewLeaseTask : renewLeaseTasks) { - try { - if (!renewLeaseTask.get()) { - lostLeases++; - } - } catch (InterruptedException e) { - log.info("Interrupted while waiting for a lease to renew."); - leasesInUnknownState += 1; - Thread.currentThread().interrupt(); - } catch (ExecutionException e) { - log.error("Encountered an exception while renewing a lease.", e.getCause()); - leasesInUnknownState += 1; - lastException = e; - } - } - - renewLeaseTaskMetricsScope.addData( - "LostLeases", lostLeases, StandardUnit.Count, MetricsLevel.SUMMARY); - renewLeaseTaskMetricsScope.addData( - "CurrentLeases", ownedLeases.size(), StandardUnit.Count, MetricsLevel.SUMMARY); - if (leasesInUnknownState > 0) { - throw new DependencyException(String.format("Encountered an exception while renewing leases. " - + "The number of leases which might not have been renewed is %d", - leasesInUnknownState), - lastException); - } - } - - private class RenewLeaseTask implements Callable { - - private final T lease; - private final IMetricsScope metricsScope; - - public RenewLeaseTask(T lease, IMetricsScope metricsScope) { - this.lease = lease; - this.metricsScope = metricsScope; - } - - @Override - public Boolean call() throws Exception { - MetricsHelper.setMetricsScope(metricsScope); - try { - return renewLease(lease); - } finally { - MetricsHelper.unsetMetricsScope(); - } - } - } - - private boolean renewLease(T lease) throws DependencyException, InvalidStateException { - return renewLease(lease, false); - } - - private boolean renewLease(T lease, boolean renewEvenIfExpired) throws DependencyException, InvalidStateException { - String leaseKey = lease.getLeaseKey(); - - boolean success = false; - boolean renewedLease = false; - long startTime = System.currentTimeMillis(); - try { - for (int i = 1; i <= RENEWAL_RETRIES; i++) { - try { - synchronized (lease) { - // Don't renew expired lease during regular renewals. getCopyOfHeldLease may have returned null - // triggering the application processing to treat this as a lost lease (fail checkpoint with - // ShutdownException). - boolean isLeaseExpired = lease.isExpired(leaseDurationNanos, System.nanoTime()); - if (renewEvenIfExpired || !isLeaseExpired) { - renewedLease = leaseManager.renewLease(lease); - } - if (renewedLease) { - lease.setLastCounterIncrementNanos(System.nanoTime()); - } - } - - if (renewedLease) { - if (log.isDebugEnabled()) { - log.debug("Worker {} successfully renewed lease with key {}", - workerIdentifier, - leaseKey); - } - } else { - log.info("Worker {} lost lease with key {}", workerIdentifier, leaseKey); - ownedLeases.remove(leaseKey); - } - - success = true; - break; - } catch (ProvisionedThroughputException e) { - log.info("Worker {} could not renew lease with key {} on try {} out of {} due to capacity", - workerIdentifier, - leaseKey, - i, - RENEWAL_RETRIES); - } - } - } finally { - MetricsHelper.addSuccessAndLatency("RenewLease", startTime, success, MetricsLevel.DETAILED); - } - - return renewedLease; - } + public Map getCurrentlyHeldLeases(); /** - * {@inheritDoc} - */ - @Override - public Map getCurrentlyHeldLeases() { - Map result = new HashMap(); - long now = System.nanoTime(); - - for (String leaseKey : ownedLeases.keySet()) { - T copy = getCopyOfHeldLease(leaseKey, now); - if (copy != null) { - result.put(copy.getLeaseKey(), copy); - } - } - - return result; - } - - /** - * {@inheritDoc} - */ - @Override - public T getCurrentlyHeldLease(String leaseKey) { - return getCopyOfHeldLease(leaseKey, System.nanoTime()); - } - - /** - * Internal method to return a lease with a specific lease key only if we currently hold it. + * @param leaseKey key of the lease to retrieve * - * @param leaseKey key of lease to return - * @param now current timestamp for old-ness checking - * @return non-authoritative copy of the held lease, or null if we don't currently hold it + * @return a deep copy of a currently held lease, or null if we don't hold the lease */ - private T getCopyOfHeldLease(String leaseKey, long now) { - T authoritativeLease = ownedLeases.get(leaseKey); - if (authoritativeLease == null) { - return null; - } else { - T copy = null; - synchronized (authoritativeLease) { - copy = authoritativeLease.copy(); - } - - if (copy.isExpired(leaseDurationNanos, now)) { - log.info("getCurrentlyHeldLease not returning lease with key {} because it is expired", - copy.getLeaseKey()); - return null; - } else { - return copy; - } - } - } + public T getCurrentlyHeldLease(String leaseKey); /** - * {@inheritDoc} + * Adds leases to this LeaseRenewer's set of currently held leases. Leases must have lastRenewalNanos set to the + * last time the lease counter was incremented before being passed to this method. + * + * @param newLeases new leases. */ - @Override - public boolean updateLease(T lease, UUID concurrencyToken) - throws DependencyException, InvalidStateException, ProvisionedThroughputException { - verifyNotNull(lease, "lease cannot be null"); - verifyNotNull(lease.getLeaseKey(), "leaseKey cannot be null"); - verifyNotNull(concurrencyToken, "concurrencyToken cannot be null"); - - String leaseKey = lease.getLeaseKey(); - T authoritativeLease = ownedLeases.get(leaseKey); - - if (authoritativeLease == null) { - log.info("Worker {} could not update lease with key {} because it does not hold it", - workerIdentifier, - leaseKey); - return false; - } - - /* - * If the passed-in concurrency token doesn't match the concurrency token of the authoritative lease, it means - * the lease was lost and regained between when the caller acquired his concurrency token and when the caller - * called update. - */ - if (!authoritativeLease.getConcurrencyToken().equals(concurrencyToken)) { - log.info("Worker {} refusing to update lease with key {} because" - + " concurrency tokens don't match", workerIdentifier, leaseKey); - return false; - } - - long startTime = System.currentTimeMillis(); - boolean success = false; - try { - synchronized (authoritativeLease) { - authoritativeLease.update(lease); - boolean updatedLease = leaseManager.updateLease(authoritativeLease); - if (updatedLease) { - // Updates increment the counter - authoritativeLease.setLastCounterIncrementNanos(System.nanoTime()); - } else { - /* - * If updateLease returns false, it means someone took the lease from us. Remove the lease - * from our set of owned leases pro-actively rather than waiting for a run of renewLeases(). - */ - log.info("Worker {} lost lease with key {} - discovered during update", - workerIdentifier, - leaseKey); - - /* - * Remove only if the value currently in the map is the same as the authoritative lease. We're - * guarding against a pause after the concurrency token check above. It plays out like so: - * - * 1) Concurrency token check passes - * 2) Pause. Lose lease, re-acquire lease. This requires at least one lease counter update. - * 3) Unpause. leaseManager.updateLease fails conditional write due to counter updates, returns - * false. - * 4) ownedLeases.remove(key, value) doesn't do anything because authoritativeLease does not - * .equals() the re-acquired version in the map on the basis of lease counter. This is what we want. - * If we just used ownedLease.remove(key), we would have pro-actively removed a lease incorrectly. - * - * Note that there is a subtlety here - Lease.equals() deliberately does not check the concurrency - * token, but it does check the lease counter, so this scheme works. - */ - ownedLeases.remove(leaseKey, authoritativeLease); - } - - success = true; - return updatedLease; - } - } finally { - MetricsHelper.addSuccessAndLatency("UpdateLease", startTime, success, MetricsLevel.DETAILED); - } - } + public void addLeasesToRenew(Collection newLeases); /** - * {@inheritDoc} + * Clears this LeaseRenewer's set of currently held leases. */ - @Override - public void addLeasesToRenew(Collection newLeases) { - verifyNotNull(newLeases, "newLeases cannot be null"); - - for (T lease : newLeases) { - if (lease.getLastCounterIncrementNanos() == null) { - log.info("addLeasesToRenew ignoring lease with key {} because it does not have lastRenewalNanos set", - lease.getLeaseKey()); - continue; - } - - T authoritativeLease = lease.copy(); - - /* - * Assign a concurrency token when we add this to the set of currently owned leases. This ensures that - * every time we acquire a lease, it gets a new concurrency token. - */ - authoritativeLease.setConcurrencyToken(UUID.randomUUID()); - ownedLeases.put(authoritativeLease.getLeaseKey(), authoritativeLease); - } - } + public void clearCurrentlyHeldLeases(); /** - * {@inheritDoc} - */ - @Override - public void clearCurrentlyHeldLeases() { - ownedLeases.clear(); - } - - /** - * {@inheritDoc} + * Stops the lease renewer from continunig to maintain the given lease. + * * @param lease the lease to drop. */ - @Override - public void dropLease(T lease) { - ownedLeases.remove(lease.getLeaseKey()); - } + void dropLease(T lease); /** - * {@inheritDoc} + * Update application-specific fields in a currently held lease. Cannot be used to update internal fields such as + * leaseCounter, leaseOwner, etc. Fails if we do not hold the lease, or if the concurrency token does not match + * the concurrency token on the internal authoritative copy of the lease (ie, if we lost and re-acquired the lease). + * + * @param lease lease object containing updated data + * @param concurrencyToken obtained by calling Lease.getConcurrencyToken for a currently held lease + * + * @return true if update succeeds, false otherwise + * + * @throws InvalidStateException if lease table does not exist + * @throws ProvisionedThroughputException if DynamoDB update fails due to lack of capacity + * @throws DependencyException if DynamoDB update fails in an unexpected way */ - @Override - public void initialize() throws DependencyException, InvalidStateException, ProvisionedThroughputException { - Collection leases = leaseManager.listLeases(); - List myLeases = new LinkedList(); - boolean renewEvenIfExpired = true; - - for (T lease : leases) { - if (workerIdentifier.equals(lease.getLeaseOwner())) { - log.info(" Worker {} found lease {}", workerIdentifier, lease); - // Okay to renew even if lease is expired, because we start with an empty list and we add the lease to - // our list only after a successful renew. So we don't need to worry about the edge case where we could - // continue renewing a lease after signaling a lease loss to the application. - if (renewLease(lease, renewEvenIfExpired)) { - myLeases.add(lease); - } - } else { - log.debug("Worker {} ignoring lease {} ", workerIdentifier, lease); - } - } - - addLeasesToRenew(myLeases); - } - - private void verifyNotNull(Object object, String message) { - if (object == null) { - throw new IllegalArgumentException(message); - } - } + boolean updateLease(T lease, UUID concurrencyToken) + throws DependencyException, InvalidStateException, ProvisionedThroughputException; } diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseTaker.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseTaker.java index 72661497..e54533b3 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseTaker.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseTaker.java @@ -14,522 +14,36 @@ */ package software.amazon.kinesis.leases; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.concurrent.Callable; -import java.util.concurrent.TimeUnit; -import com.amazonaws.services.cloudwatch.model.StandardUnit; import software.amazon.kinesis.leases.exceptions.DependencyException; import software.amazon.kinesis.leases.exceptions.InvalidStateException; -import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException; -import software.amazon.kinesis.metrics.MetricsHelper; -import software.amazon.kinesis.metrics.IMetricsScope; -import software.amazon.kinesis.metrics.MetricsLevel; - -import lombok.extern.slf4j.Slf4j; +import software.amazon.kinesis.leases.Lease; /** - * An implementation of ILeaseTaker that uses DynamoDB via LeaseManager. + * ILeaseTaker is used by LeaseCoordinator to take new leases, or leases that other workers fail to renew. Each + * LeaseCoordinator instance corresponds to one worker and uses exactly one ILeaseTaker to take leases for that worker. */ -@Slf4j -public class LeaseTaker implements ILeaseTaker { - private static final int TAKE_RETRIES = 3; - private static final int SCAN_RETRIES = 1; - - // See note on takeLeases(Callable) for why we have this callable. - private static final Callable SYSTEM_CLOCK_CALLABLE = new Callable() { - - @Override - public Long call() { - return System.nanoTime(); - } - }; - - private final ILeaseManager leaseManager; - private final String workerIdentifier; - private final Map allLeases = new HashMap(); - private final long leaseDurationNanos; - private int maxLeasesForWorker = Integer.MAX_VALUE; - private int maxLeasesToStealAtOneTime = 1; - - private long lastScanTimeNanos = 0L; - - public LeaseTaker(ILeaseManager leaseManager, String workerIdentifier, long leaseDurationMillis) { - this.leaseManager = leaseManager; - this.workerIdentifier = workerIdentifier; - this.leaseDurationNanos = TimeUnit.MILLISECONDS.toNanos(leaseDurationMillis); - } +public interface LeaseTaker { /** - * Worker will not acquire more than the specified max number of leases even if there are more - * shards that need to be processed. This can be used in scenarios where a worker is resource constrained or - * to prevent lease thrashing when small number of workers pick up all leases for small amount of time during - * deployment. - * Note that setting a low value may cause data loss (e.g. if there aren't enough Workers to make progress on all - * shards). When setting the value for this property, one must ensure enough workers are present to process - * shards and should consider future resharding, child shards that may be blocked on parent shards, some workers - * becoming unhealthy, etc. - * - * @param maxLeasesForWorker Max leases this Worker can handle at a time - * @return LeaseTaker + * Compute the set of leases available to be taken and attempt to take them. Lease taking rules are: + * + * 1) If a lease's counter hasn't changed in long enough, try to take it. + * 2) If we see a lease we've never seen before, take it only if owner == null. If it's owned, odds are the owner is + * holding it. We can't tell until we see it more than once. + * 3) For load balancing purposes, you may violate rules 1 and 2 for EXACTLY ONE lease per call of takeLeases(). + * + * @return map of shardId to Lease object for leases we just successfully took. + * + * @throws DependencyException on unexpected DynamoDB failures + * @throws InvalidStateException if lease table does not exist */ - public LeaseTaker withMaxLeasesForWorker(int maxLeasesForWorker) { - if (maxLeasesForWorker <= 0) { - throw new IllegalArgumentException("maxLeasesForWorker should be >= 1"); - } - this.maxLeasesForWorker = maxLeasesForWorker; - return this; - } + public abstract Map takeLeases() throws DependencyException, InvalidStateException; /** - * Max leases to steal from a more loaded Worker at one time (for load balancing). - * Setting this to a higher number can allow for faster load convergence (e.g. during deployments, cold starts), - * but can cause higher churn in the system. - * - * @param maxLeasesToStealAtOneTime Steal up to this many leases at one time (for load balancing) - * @return LeaseTaker + * @return workerIdentifier for this LeaseTaker */ - public LeaseTaker withMaxLeasesToStealAtOneTime(int maxLeasesToStealAtOneTime) { - if (maxLeasesToStealAtOneTime <= 0) { - throw new IllegalArgumentException("maxLeasesToStealAtOneTime should be >= 1"); - } - this.maxLeasesToStealAtOneTime = maxLeasesToStealAtOneTime; - return this; - } + public abstract String getWorkerIdentifier(); - /** - * {@inheritDoc} - */ - @Override - public Map takeLeases() throws DependencyException, InvalidStateException { - return takeLeases(SYSTEM_CLOCK_CALLABLE); - } - - /** - * Internal implementation of takeLeases. Takes a callable that can provide the time to enable test cases without - * Thread.sleep. Takes a callable instead of a raw time value because the time needs to be computed as-of - * immediately after the scan. - * - * @param timeProvider Callable that will supply the time - * - * @return map of lease key to taken lease - * - * @throws DependencyException - * @throws InvalidStateException - */ - synchronized Map takeLeases(Callable timeProvider) - throws DependencyException, InvalidStateException { - // Key is leaseKey - Map takenLeases = new HashMap(); - - long startTime = System.currentTimeMillis(); - boolean success = false; - - ProvisionedThroughputException lastException = null; - - try { - for (int i = 1; i <= SCAN_RETRIES; i++) { - try { - updateAllLeases(timeProvider); - success = true; - } catch (ProvisionedThroughputException e) { - log.info("Worker {} could not find expired leases on try {} out of {}", - workerIdentifier, - i, - TAKE_RETRIES); - lastException = e; - } - } - } finally { - MetricsHelper.addSuccessAndLatency("ListLeases", startTime, success, MetricsLevel.DETAILED); - } - - if (lastException != null) { - log.error("Worker {} could not scan leases table, aborting takeLeases. Exception caught by last retry:", - workerIdentifier, - lastException); - return takenLeases; - } - - List expiredLeases = getExpiredLeases(); - - Set leasesToTake = computeLeasesToTake(expiredLeases); - Set untakenLeaseKeys = new HashSet(); - - for (T lease : leasesToTake) { - String leaseKey = lease.getLeaseKey(); - - startTime = System.currentTimeMillis(); - success = false; - try { - for (int i = 1; i <= TAKE_RETRIES; i++) { - try { - if (leaseManager.takeLease(lease, workerIdentifier)) { - lease.setLastCounterIncrementNanos(System.nanoTime()); - takenLeases.put(leaseKey, lease); - } else { - untakenLeaseKeys.add(leaseKey); - } - - success = true; - break; - } catch (ProvisionedThroughputException e) { - log.info("Could not take lease with key {} for worker {} on try {} out of {} due to capacity", - leaseKey, - workerIdentifier, - i, - TAKE_RETRIES); - } - } - } finally { - MetricsHelper.addSuccessAndLatency("TakeLease", startTime, success, MetricsLevel.DETAILED); - } - } - - if (takenLeases.size() > 0) { - log.info("Worker {} successfully took {} leases: {}", - workerIdentifier, - takenLeases.size(), - stringJoin(takenLeases.keySet(), ", ")); - } - - if (untakenLeaseKeys.size() > 0) { - log.info("Worker {} failed to take {} leases: {}", - workerIdentifier, - untakenLeaseKeys.size(), - stringJoin(untakenLeaseKeys, ", ")); - } - - MetricsHelper.getMetricsScope().addData( - "TakenLeases", takenLeases.size(), StandardUnit.Count, MetricsLevel.SUMMARY); - - return takenLeases; - } - - /** Package access for testing purposes. - * - * @param strings - * @param delimiter - * @return Joined string. - */ - static String stringJoin(Collection strings, String delimiter) { - StringBuilder builder = new StringBuilder(); - boolean needDelimiter = false; - for (String string : strings) { - if (needDelimiter) { - builder.append(delimiter); - } - builder.append(string); - needDelimiter = true; - } - - return builder.toString(); - } - - /** - * Scan all leases and update lastRenewalTime. Add new leases and delete old leases. - * - * @param timeProvider callable that supplies the current time - * - * @return list of expired leases, possibly empty, never null. - * - * @throws ProvisionedThroughputException if listLeases fails due to lack of provisioned throughput - * @throws InvalidStateException if the lease table does not exist - * @throws DependencyException if listLeases fails in an unexpected way - */ - private void updateAllLeases(Callable timeProvider) - throws DependencyException, InvalidStateException, ProvisionedThroughputException { - List freshList = leaseManager.listLeases(); - try { - lastScanTimeNanos = timeProvider.call(); - } catch (Exception e) { - throw new DependencyException("Exception caught from timeProvider", e); - } - - // This set will hold the lease keys not updated by the previous listLeases call. - Set notUpdated = new HashSet(allLeases.keySet()); - - // Iterate over all leases, finding ones to try to acquire that haven't changed since the last iteration - for (T lease : freshList) { - String leaseKey = lease.getLeaseKey(); - - T oldLease = allLeases.get(leaseKey); - allLeases.put(leaseKey, lease); - notUpdated.remove(leaseKey); - - if (oldLease != null) { - // If we've seen this lease before... - if (oldLease.getLeaseCounter().equals(lease.getLeaseCounter())) { - // ...and the counter hasn't changed, propagate the lastRenewalNanos time from the old lease - lease.setLastCounterIncrementNanos(oldLease.getLastCounterIncrementNanos()); - } else { - // ...and the counter has changed, set lastRenewalNanos to the time of the scan. - lease.setLastCounterIncrementNanos(lastScanTimeNanos); - } - } else { - if (lease.getLeaseOwner() == null) { - // if this new lease is unowned, it's never been renewed. - lease.setLastCounterIncrementNanos(0L); - - if (log.isDebugEnabled()) { - log.debug("Treating new lease with key {} as never renewed because it is new and unowned.", - leaseKey); - } - } else { - // if this new lease is owned, treat it as renewed as of the scan - lease.setLastCounterIncrementNanos(lastScanTimeNanos); - if (log.isDebugEnabled()) { - log.debug("Treating new lease with key {} as recently renewed because it is new and owned.", - leaseKey); - } - } - } - } - - // Remove dead leases from allLeases - for (String key : notUpdated) { - allLeases.remove(key); - } - } - - /** - * @return list of leases that were expired as of our last scan. - */ - private List getExpiredLeases() { - List expiredLeases = new ArrayList(); - - for (T lease : allLeases.values()) { - if (lease.isExpired(leaseDurationNanos, lastScanTimeNanos)) { - expiredLeases.add(lease); - } - } - - return expiredLeases; - } - - /** - * Compute the number of leases I should try to take based on the state of the system. - * - * @param allLeases map of shardId to lease containing all leases - * @param expiredLeases list of leases we determined to be expired - * @return set of leases to take. - */ - private Set computeLeasesToTake(List expiredLeases) { - Map leaseCounts = computeLeaseCounts(expiredLeases); - Set leasesToTake = new HashSet(); - IMetricsScope metrics = MetricsHelper.getMetricsScope(); - - int numLeases = allLeases.size(); - int numWorkers = leaseCounts.size(); - - if (numLeases == 0) { - // If there are no leases, I shouldn't try to take any. - return leasesToTake; - } - - int target; - if (numWorkers >= numLeases) { - // If we have n leases and n or more workers, each worker can have up to 1 lease, including myself. - target = 1; - } else { - /* - * numWorkers must be < numLeases. - * - * Our target for each worker is numLeases / numWorkers (+1 if numWorkers doesn't evenly divide numLeases) - */ - target = numLeases / numWorkers + (numLeases % numWorkers == 0 ? 0 : 1); - - // Spill over is the number of leases this worker should have claimed, but did not because it would - // exceed the max allowed for this worker. - int leaseSpillover = Math.max(0, target - maxLeasesForWorker); - if (target > maxLeasesForWorker) { - log.warn("Worker {} target is {} leases and maxLeasesForWorker is {}." - + " Resetting target to {}, lease spillover is {}. " - + " Note that some shards may not be processed if no other workers are able to pick them up.", - workerIdentifier, - target, - maxLeasesForWorker, - maxLeasesForWorker, - leaseSpillover); - target = maxLeasesForWorker; - } - metrics.addData("LeaseSpillover", leaseSpillover, StandardUnit.Count, MetricsLevel.SUMMARY); - } - - int myCount = leaseCounts.get(workerIdentifier); - int numLeasesToReachTarget = target - myCount; - - if (numLeasesToReachTarget <= 0) { - // If we don't need anything, return the empty set. - return leasesToTake; - } - - // Shuffle expiredLeases so workers don't all try to contend for the same leases. - Collections.shuffle(expiredLeases); - - int originalExpiredLeasesSize = expiredLeases.size(); - if (expiredLeases.size() > 0) { - // If we have expired leases, get up to leases from expiredLeases - for (; numLeasesToReachTarget > 0 && expiredLeases.size() > 0; numLeasesToReachTarget--) { - leasesToTake.add(expiredLeases.remove(0)); - } - } else { - // If there are no expired leases and we need a lease, consider stealing. - List leasesToSteal = chooseLeasesToSteal(leaseCounts, numLeasesToReachTarget, target); - for (T leaseToSteal : leasesToSteal) { - log.info("Worker {} needed {} leases but none were expired, so it will steal lease {} from {}", - workerIdentifier, - numLeasesToReachTarget, - leaseToSteal.getLeaseKey(), - leaseToSteal.getLeaseOwner()); - leasesToTake.add(leaseToSteal); - } - } - - if (!leasesToTake.isEmpty()) { - log.info("Worker {} saw {} total leases, {} available leases, {} " - + "workers. Target is {} leases, I have {} leases, I will take {} leases", - workerIdentifier, - numLeases, - originalExpiredLeasesSize, - numWorkers, - target, - myCount, - leasesToTake.size()); - } - - metrics.addData("TotalLeases", numLeases, StandardUnit.Count, MetricsLevel.DETAILED); - metrics.addData("ExpiredLeases", originalExpiredLeasesSize, StandardUnit.Count, MetricsLevel.SUMMARY); - metrics.addData("NumWorkers", numWorkers, StandardUnit.Count, MetricsLevel.SUMMARY); - metrics.addData("NeededLeases", numLeasesToReachTarget, StandardUnit.Count, MetricsLevel.DETAILED); - metrics.addData("LeasesToTake", leasesToTake.size(), StandardUnit.Count, MetricsLevel.DETAILED); - - return leasesToTake; - } - - /** - * Choose leases to steal by randomly selecting one or more (up to max) from the most loaded worker. - * Stealing rules: - * - * Steal up to maxLeasesToStealAtOneTime leases from the most loaded worker if - * a) he has > target leases and I need >= 1 leases : steal min(leases needed, maxLeasesToStealAtOneTime) - * b) he has == target leases and I need > 1 leases : steal 1 - * - * @param leaseCounts map of workerIdentifier to lease count - * @param needed # of leases needed to reach the target leases for the worker - * @param target target # of leases per worker - * @return Leases to steal, or empty list if we should not steal - */ - private List chooseLeasesToSteal(Map leaseCounts, int needed, int target) { - List leasesToSteal = new ArrayList<>(); - - Entry mostLoadedWorker = null; - // Find the most loaded worker - for (Entry worker : leaseCounts.entrySet()) { - if (mostLoadedWorker == null || mostLoadedWorker.getValue() < worker.getValue()) { - mostLoadedWorker = worker; - } - } - - int numLeasesToSteal = 0; - if ((mostLoadedWorker.getValue() >= target) && (needed > 0)) { - int leasesOverTarget = mostLoadedWorker.getValue() - target; - numLeasesToSteal = Math.min(needed, leasesOverTarget); - // steal 1 if we need > 1 and max loaded worker has target leases. - if ((needed > 1) && (numLeasesToSteal == 0)) { - numLeasesToSteal = 1; - } - numLeasesToSteal = Math.min(numLeasesToSteal, maxLeasesToStealAtOneTime); - } - - if (numLeasesToSteal <= 0) { - if (log.isDebugEnabled()) { - log.debug(String.format("Worker %s not stealing from most loaded worker %s. He has %d," - + " target is %d, and I need %d", - workerIdentifier, - mostLoadedWorker.getKey(), - mostLoadedWorker.getValue(), - target, - needed)); - } - return leasesToSteal; - } else { - if (log.isDebugEnabled()) { - log.debug("Worker {} will attempt to steal {} leases from most loaded worker {}. " - + " He has {} leases, target is {}, I need {}, maxLeasesToSteatAtOneTime is {}.", - workerIdentifier, - numLeasesToSteal, - mostLoadedWorker.getKey(), - mostLoadedWorker.getValue(), - target, - needed, - maxLeasesToStealAtOneTime); - } - } - - String mostLoadedWorkerIdentifier = mostLoadedWorker.getKey(); - List candidates = new ArrayList(); - // Collect leases belonging to that worker - for (T lease : allLeases.values()) { - if (mostLoadedWorkerIdentifier.equals(lease.getLeaseOwner())) { - candidates.add(lease); - } - } - - // Return random ones - Collections.shuffle(candidates); - int toIndex = Math.min(candidates.size(), numLeasesToSteal); - leasesToSteal.addAll(candidates.subList(0, toIndex)); - - return leasesToSteal; - } - - /** - * Count leases by host. Always includes myself, but otherwise only includes hosts that are currently holding - * leases. - * - * @param expiredLeases list of leases that are currently expired - * @return map of workerIdentifier to lease count - */ - private Map computeLeaseCounts(List expiredLeases) { - Map leaseCounts = new HashMap(); - - // Compute the number of leases per worker by looking through allLeases and ignoring leases that have expired. - for (T lease : allLeases.values()) { - if (!expiredLeases.contains(lease)) { - String leaseOwner = lease.getLeaseOwner(); - Integer oldCount = leaseCounts.get(leaseOwner); - if (oldCount == null) { - leaseCounts.put(leaseOwner, 1); - } else { - leaseCounts.put(leaseOwner, oldCount + 1); - } - } - } - - // If I have no leases, I wasn't represented in leaseCounts. Let's fix that. - Integer myCount = leaseCounts.get(workerIdentifier); - if (myCount == null) { - myCount = 0; - leaseCounts.put(workerIdentifier, myCount); - } - - return leaseCounts; - } - - /** - * {@inheritDoc} - */ - @Override - public String getWorkerIdentifier() { - return workerIdentifier; - } } diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/InitializationInput.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/InitializationInput.java index 90d4a7f9..ac5951ee 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/InitializationInput.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/InitializationInput.java @@ -14,12 +14,12 @@ */ package software.amazon.kinesis.lifecycle; -import software.amazon.kinesis.processor.IRecordProcessor; +import software.amazon.kinesis.processor.RecordProcessor; import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; /** * Container for the parameters to the IRecordProcessor's - * {@link IRecordProcessor#initialize(InitializationInput + * {@link RecordProcessor#initialize(InitializationInput * initializationInput) initialize} method. */ public class InitializationInput { diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/InitializeTask.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/InitializeTask.java index 85983f17..4d37b796 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/InitializeTask.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/InitializeTask.java @@ -17,10 +17,10 @@ package software.amazon.kinesis.lifecycle; import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStreamExtended; import lombok.NonNull; import lombok.RequiredArgsConstructor; -import software.amazon.kinesis.coordinator.RecordProcessorCheckpointer; +import software.amazon.kinesis.checkpoint.RecordProcessorCheckpointer; import software.amazon.kinesis.leases.ShardInfo; -import software.amazon.kinesis.processor.ICheckpoint; -import software.amazon.kinesis.processor.IRecordProcessor; +import software.amazon.kinesis.processor.Checkpointer; +import software.amazon.kinesis.processor.RecordProcessor; import software.amazon.kinesis.checkpoint.Checkpoint; import software.amazon.kinesis.retrieval.GetRecordsCache; import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; @@ -40,9 +40,9 @@ public class InitializeTask implements ITask { @NonNull private final ShardInfo shardInfo; @NonNull - private final IRecordProcessor recordProcessor; + private final RecordProcessor recordProcessor; @NonNull - private final ICheckpoint checkpoint; + private final Checkpointer checkpoint; @NonNull private final RecordProcessorCheckpointer recordProcessorCheckpointer; @NonNull @@ -69,7 +69,7 @@ public class InitializeTask implements ITask { try { log.debug("Initializing ShardId {}", shardInfo); Checkpoint initialCheckpointObject = checkpoint.getCheckpointObject(shardInfo.shardId()); - ExtendedSequenceNumber initialCheckpoint = initialCheckpointObject.getCheckpoint(); + ExtendedSequenceNumber initialCheckpoint = initialCheckpointObject.checkpoint(); cache.start(initialCheckpoint, initialPositionInStream); @@ -80,7 +80,7 @@ public class InitializeTask implements ITask { final InitializationInput initializationInput = new InitializationInput() .withShardId(shardInfo.shardId()) .withExtendedSequenceNumber(initialCheckpoint) - .withPendingCheckpointSequenceNumber(initialCheckpointObject.getPendingCheckpoint()); + .withPendingCheckpointSequenceNumber(initialCheckpointObject.pendingCheckpoint()); final long recordProcessorStartTimeMillis = System.currentTimeMillis(); try { recordProcessor.initialize(initializationInput); diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ProcessRecordsInput.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ProcessRecordsInput.java index 96008359..acdb8b75 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ProcessRecordsInput.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ProcessRecordsInput.java @@ -23,11 +23,11 @@ import software.amazon.kinesis.processor.IRecordProcessorCheckpointer; import com.amazonaws.services.kinesis.model.Record; import lombok.Getter; -import software.amazon.kinesis.processor.IRecordProcessor; +import software.amazon.kinesis.processor.RecordProcessor; /** * Container for the parameters to the IRecordProcessor's - * {@link IRecordProcessor#processRecords( + * {@link RecordProcessor#processRecords( * ProcessRecordsInput processRecordsInput) processRecords} method. */ @AllArgsConstructor diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ProcessTask.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ProcessTask.java index 46fde1d0..612c5c53 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ProcessTask.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ProcessTask.java @@ -19,13 +19,13 @@ import com.amazonaws.services.kinesis.model.Shard; import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import software.amazon.kinesis.coordinator.RecordProcessorCheckpointer; +import software.amazon.kinesis.checkpoint.RecordProcessorCheckpointer; import software.amazon.kinesis.leases.LeaseManagerProxy; import software.amazon.kinesis.leases.ShardInfo; import software.amazon.kinesis.metrics.IMetricsScope; import software.amazon.kinesis.metrics.MetricsHelper; import software.amazon.kinesis.metrics.MetricsLevel; -import software.amazon.kinesis.processor.IRecordProcessor; +import software.amazon.kinesis.processor.RecordProcessor; import software.amazon.kinesis.retrieval.GetRecordsCache; import software.amazon.kinesis.retrieval.ThrottlingReporter; import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; @@ -42,7 +42,7 @@ public class ProcessTask implements ITask { private static final String RECORD_PROCESSOR_PROCESS_RECORDS_METRIC = "RecordProcessor.processRecords"; private final ShardInfo shardInfo; - private final IRecordProcessor recordProcessor; + private final RecordProcessor recordProcessor; private final RecordProcessorCheckpointer recordProcessorCheckpointer; private final TaskType taskType = TaskType.PROCESS; private final long backoffTimeMillis; @@ -73,7 +73,7 @@ public class ProcessTask implements ITask { } public ProcessTask(@NonNull final ShardInfo shardInfo, - @NonNull final IRecordProcessor recordProcessor, + @NonNull final RecordProcessor recordProcessor, @NonNull final RecordProcessorCheckpointer recordProcessorCheckpointer, final long backoffTimeMillis, final boolean skipShardSyncAtWorkerInitializationIfLeasesExist, diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/RecordProcessorShim.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/RecordProcessorShim.java index bd5ab34f..644f15fa 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/RecordProcessorShim.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/RecordProcessorShim.java @@ -20,14 +20,14 @@ import software.amazon.kinesis.lifecycle.events.RecordsReceived; import software.amazon.kinesis.lifecycle.events.ShardCompleted; import software.amazon.kinesis.lifecycle.events.ShutdownRequested; import software.amazon.kinesis.lifecycle.events.Started; -import software.amazon.kinesis.processor.IRecordProcessor; +import software.amazon.kinesis.processor.RecordProcessor; import software.amazon.kinesis.processor.IRecordProcessorCheckpointer; -import software.amazon.kinesis.processor.IShutdownNotificationAware; +import software.amazon.kinesis.processor.ShutdownNotificationAware; @RequiredArgsConstructor public class RecordProcessorShim implements RecordProcessorLifecycle { - private final IRecordProcessor delegate; + private final RecordProcessor delegate; @Override public void started(Started started) { @@ -60,8 +60,8 @@ public class RecordProcessorShim implements RecordProcessorLifecycle { @Override public void shutdownRequested(ShutdownRequested shutdownRequested) { - if (delegate instanceof IShutdownNotificationAware) { - IShutdownNotificationAware aware = (IShutdownNotificationAware)delegate; + if (delegate instanceof ShutdownNotificationAware) { + ShutdownNotificationAware aware = (ShutdownNotificationAware)delegate; aware.shutdownRequested(shutdownRequested.getCheckpointer()); } } diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShardConsumer.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShardConsumer.java index f3543f74..3dbcf5a3 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShardConsumer.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShardConsumer.java @@ -34,16 +34,15 @@ import lombok.RequiredArgsConstructor; import lombok.Synchronized; import lombok.experimental.Accessors; import lombok.extern.slf4j.Slf4j; -import software.amazon.kinesis.coordinator.RecordProcessorCheckpointer; +import software.amazon.kinesis.checkpoint.RecordProcessorCheckpointer; import software.amazon.kinesis.leases.ILeaseManager; import software.amazon.kinesis.leases.KinesisClientLease; -import software.amazon.kinesis.leases.LeaseManager; import software.amazon.kinesis.leases.LeaseManagerProxy; import software.amazon.kinesis.leases.ShardInfo; import software.amazon.kinesis.metrics.IMetricsFactory; import software.amazon.kinesis.metrics.MetricsCollectingTaskDecorator; -import software.amazon.kinesis.processor.ICheckpoint; -import software.amazon.kinesis.processor.IRecordProcessor; +import software.amazon.kinesis.processor.Checkpointer; +import software.amazon.kinesis.processor.RecordProcessor; import software.amazon.kinesis.retrieval.GetRecordsCache; /** @@ -67,9 +66,9 @@ public class ShardConsumer { @NonNull private final GetRecordsCache getRecordsCache; @NonNull - private final IRecordProcessor recordProcessor; + private final RecordProcessor recordProcessor; @NonNull - private final ICheckpoint checkpoint; + private final Checkpointer checkpoint; @NonNull private final RecordProcessorCheckpointer recordProcessorCheckpointer; private final long parentShardPollIntervalMillis; diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShardConsumerShutdownNotification.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShardConsumerShutdownNotification.java index 1dbfd504..fa423cca 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShardConsumerShutdownNotification.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShardConsumerShutdownNotification.java @@ -16,7 +16,7 @@ package software.amazon.kinesis.lifecycle; import java.util.concurrent.CountDownLatch; -import software.amazon.kinesis.processor.IShutdownNotificationAware; +import software.amazon.kinesis.processor.ShutdownNotificationAware; import software.amazon.kinesis.leases.KinesisClientLease; import software.amazon.kinesis.leases.LeaseCoordinator; @@ -43,7 +43,7 @@ public class ShardConsumerShutdownNotification implements ShutdownNotification { * the lease that this shutdown request will free once initial shutdown is complete * @param notificationCompleteLatch * used to inform the caller once the - * {@link IShutdownNotificationAware} object has been + * {@link ShutdownNotificationAware} object has been * notified of the shutdown request. * @param shutdownCompleteLatch * used to inform the caller once the record processor is fully shutdown diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShutdownInput.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShutdownInput.java index 958693ae..dd817046 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShutdownInput.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShutdownInput.java @@ -16,12 +16,12 @@ package software.amazon.kinesis.lifecycle; import lombok.Data; import lombok.experimental.Accessors; -import software.amazon.kinesis.processor.IRecordProcessor; +import software.amazon.kinesis.processor.RecordProcessor; import software.amazon.kinesis.processor.IRecordProcessorCheckpointer; /** * Container for the parameters to the IRecordProcessor's - * {@link IRecordProcessor#shutdown(ShutdownInput + * {@link RecordProcessor#shutdown(ShutdownInput * shutdownInput) shutdown} method. */ @Data diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShutdownNotification.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShutdownNotification.java index e562c068..04c6f627 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShutdownNotification.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShutdownNotification.java @@ -14,7 +14,7 @@ */ package software.amazon.kinesis.lifecycle; -import software.amazon.kinesis.processor.IRecordProcessor; +import software.amazon.kinesis.processor.RecordProcessor; /** * A shutdown request to the ShardConsumer @@ -29,7 +29,7 @@ public interface ShutdownNotification { /** * Used to indicate that the record processor has completed the call to - * {@link IRecordProcessor#shutdown(ShutdownInput)} has + * {@link RecordProcessor#shutdown(ShutdownInput)} has * completed. */ void shutdownComplete(); diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShutdownNotificationTask.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShutdownNotificationTask.java index 5b88181b..39d57e4d 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShutdownNotificationTask.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShutdownNotificationTask.java @@ -19,8 +19,8 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import software.amazon.kinesis.leases.ShardInfo; import software.amazon.kinesis.processor.IRecordProcessorCheckpointer; -import software.amazon.kinesis.processor.IRecordProcessor; -import software.amazon.kinesis.processor.IShutdownNotificationAware; +import software.amazon.kinesis.processor.RecordProcessor; +import software.amazon.kinesis.processor.ShutdownNotificationAware; /** * Notifies record processor of incoming shutdown request, and gives them a chance to checkpoint. @@ -28,7 +28,7 @@ import software.amazon.kinesis.processor.IShutdownNotificationAware; @RequiredArgsConstructor(access = AccessLevel.PACKAGE) @Slf4j public class ShutdownNotificationTask implements ITask { - private final IRecordProcessor recordProcessor; + private final RecordProcessor recordProcessor; private final IRecordProcessorCheckpointer recordProcessorCheckpointer; private final ShutdownNotification shutdownNotification; // TODO: remove if not used @@ -38,8 +38,8 @@ public class ShutdownNotificationTask implements ITask { @Override public TaskResult call() { try { - if (recordProcessor instanceof IShutdownNotificationAware) { - IShutdownNotificationAware shutdownNotificationAware = (IShutdownNotificationAware) recordProcessor; + if (recordProcessor instanceof ShutdownNotificationAware) { + ShutdownNotificationAware shutdownNotificationAware = (ShutdownNotificationAware) recordProcessor; try { shutdownNotificationAware.shutdownRequested(recordProcessorCheckpointer); } catch (Exception ex) { diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShutdownReason.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShutdownReason.java index eae8446c..b145a3b5 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShutdownReason.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShutdownReason.java @@ -17,7 +17,7 @@ package software.amazon.kinesis.lifecycle; import lombok.AccessLevel; import lombok.Getter; import lombok.experimental.Accessors; -import software.amazon.kinesis.processor.IRecordProcessor; +import software.amazon.kinesis.processor.RecordProcessor; import static software.amazon.kinesis.lifecycle.ConsumerStates.ConsumerState; import static software.amazon.kinesis.lifecycle.ConsumerStates.ShardConsumerState; @@ -50,7 +50,7 @@ public enum ShutdownReason { /** * Indicates that the entire application is being shutdown, and if desired the record processor will be given a * final chance to checkpoint. This state will not trigger a direct call to - * {@link IRecordProcessor#shutdown(ShutdownInput)}, but + * {@link RecordProcessor#shutdown(ShutdownInput)}, but * instead depend on a different interface for backward compatibility. */ REQUESTED(1, ShardConsumerState.SHUTDOWN_REQUESTED.getConsumerState()); diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShutdownTask.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShutdownTask.java index 1051a3cb..dc19ddf1 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShutdownTask.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShutdownTask.java @@ -20,7 +20,7 @@ import com.google.common.annotations.VisibleForTesting; import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import software.amazon.kinesis.coordinator.RecordProcessorCheckpointer; +import software.amazon.kinesis.checkpoint.RecordProcessorCheckpointer; import software.amazon.kinesis.leases.ILeaseManager; import software.amazon.kinesis.leases.KinesisClientLease; import software.amazon.kinesis.leases.LeaseManagerProxy; @@ -28,7 +28,7 @@ import software.amazon.kinesis.leases.ShardInfo; import software.amazon.kinesis.leases.ShardSyncer; import software.amazon.kinesis.metrics.MetricsHelper; import software.amazon.kinesis.metrics.MetricsLevel; -import software.amazon.kinesis.processor.IRecordProcessor; +import software.amazon.kinesis.processor.RecordProcessor; import software.amazon.kinesis.retrieval.GetRecordsCache; import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; @@ -45,7 +45,7 @@ public class ShutdownTask implements ITask { @NonNull private final LeaseManagerProxy leaseManagerProxy; @NonNull - private final IRecordProcessor recordProcessor; + private final RecordProcessor recordProcessor; @NonNull private final RecordProcessorCheckpointer recordProcessorCheckpointer; @NonNull diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/metrics/MetricsConfig.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/metrics/MetricsConfig.java index a43fa541..d716eb16 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/metrics/MetricsConfig.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/metrics/MetricsConfig.java @@ -51,7 +51,6 @@ public class MetricsConfig { * * @return {@link AmazonCloudWatch} */ - @NonNull private final AmazonCloudWatch amazonCloudWatch; /** diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/ICheckpoint.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/Checkpointer.java similarity index 99% rename from amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/ICheckpoint.java rename to amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/Checkpointer.java index 4e610b43..6340eda4 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/ICheckpoint.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/Checkpointer.java @@ -21,7 +21,7 @@ import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; /** * Interface for checkpoint trackers. */ -public interface ICheckpoint { +public interface Checkpointer { /** * Record a checkpoint for a shard (e.g. sequence and subsequence numbers of last record processed diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/IPreparedCheckpointer.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/IPreparedCheckpointer.java index da06843b..faaa472e 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/IPreparedCheckpointer.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/IPreparedCheckpointer.java @@ -30,7 +30,7 @@ public interface IPreparedCheckpointer { /** * @return sequence number of pending checkpoint */ - ExtendedSequenceNumber getPendingCheckpoint(); + ExtendedSequenceNumber pendingCheckpoint(); /** * This method will record a pending checkpoint. diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/ProcessorFactory.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/ProcessorFactory.java index 0408a2b8..5fbafc15 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/ProcessorFactory.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/ProcessorFactory.java @@ -19,5 +19,5 @@ package software.amazon.kinesis.processor; * */ public interface ProcessorFactory { - IRecordProcessor createRecordProcessor(); + RecordProcessor createRecordProcessor(); } diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/IRecordProcessor.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/RecordProcessor.java similarity index 98% rename from amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/IRecordProcessor.java rename to amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/RecordProcessor.java index 4bc9a7cf..31a86037 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/IRecordProcessor.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/RecordProcessor.java @@ -23,7 +23,7 @@ import software.amazon.kinesis.lifecycle.ShutdownReason; * The Amazon Kinesis Client Library will instantiate record processors to process data records fetched from Amazon * Kinesis. */ -public interface IRecordProcessor { +public interface RecordProcessor { /** * Invoked by the Amazon Kinesis Client Library before data records are delivered to the RecordProcessor instance diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/IRecordProcessorFactory.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/RecordProcessorFactory.java similarity index 92% rename from amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/IRecordProcessorFactory.java rename to amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/RecordProcessorFactory.java index cacf0b5b..58366d31 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/IRecordProcessorFactory.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/RecordProcessorFactory.java @@ -19,13 +19,13 @@ package software.amazon.kinesis.processor; * The Amazon Kinesis Client Library will use this to instantiate a record processor per shard. * Clients may choose to create separate instantiations, or re-use instantiations. */ -public interface IRecordProcessorFactory { +public interface RecordProcessorFactory { /** * Returns a record processor to be used for processing data records for a (assigned) shard. * * @return Returns a processor object. */ - IRecordProcessor createProcessor(); + RecordProcessor createProcessor(); } diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/IShutdownNotificationAware.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/ShutdownNotificationAware.java similarity index 96% rename from amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/IShutdownNotificationAware.java rename to amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/ShutdownNotificationAware.java index a3e780ae..4f878d3f 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/IShutdownNotificationAware.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/ShutdownNotificationAware.java @@ -19,7 +19,7 @@ import software.amazon.kinesis.processor.IRecordProcessorCheckpointer; /** * Allows a record processor to indicate it's aware of requested shutdowns, and handle the request. */ -public interface IShutdownNotificationAware { +public interface ShutdownNotificationAware { /** * Called when the worker has been requested to shutdown, and gives the record processor a chance to checkpoint. diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/retrieval/SynchronousBlockingRetrievalFactory.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/retrieval/SynchronousBlockingRetrievalFactory.java index 8307159a..42092f20 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/retrieval/SynchronousBlockingRetrievalFactory.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/retrieval/SynchronousBlockingRetrievalFactory.java @@ -26,11 +26,6 @@ import software.amazon.kinesis.metrics.IMetricsFactory; */ @Data public class SynchronousBlockingRetrievalFactory implements RetrievalFactory { -// Need to remove this. Has no use any longer. - private static final long DESCRIBE_STREAM_BACKOFF_TIME_IN_MILLIS = 1500L; -// Need to remove this. Has no use any longer. - private static final int MAX_DESCRIBE_STREAM_RETRY_ATTEMPTS = 50; - @NonNull private final String streamName; @NonNull diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/retrieval/SynchronousPrefetchingRetrievalFactory.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/retrieval/SynchronousPrefetchingRetrievalFactory.java new file mode 100644 index 00000000..4ed90d17 --- /dev/null +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/retrieval/SynchronousPrefetchingRetrievalFactory.java @@ -0,0 +1,60 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.kinesis.retrieval; + +import com.amazonaws.services.kinesis.AmazonKinesis; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import software.amazon.kinesis.leases.ShardInfo; +import software.amazon.kinesis.metrics.IMetricsFactory; + +import java.util.concurrent.ExecutorService; + +/** + * + */ +@RequiredArgsConstructor +public class SynchronousPrefetchingRetrievalFactory implements RetrievalFactory { + @NonNull + private final String streamName; + @NonNull + private final AmazonKinesis amazonKinesis; + private final int maxRecords; + private final int maxPendingProcessRecordsInput; + private final int maxByteSize; + private final int maxRecordsCount; + private final int maxRecordsPerCall; + @NonNull + private final ExecutorService executorService; + private final long idleMillisBetweenCalls; + @NonNull + private final IMetricsFactory metricsFactory; + + + + @Override + public GetRecordsRetrievalStrategy createGetRecordsRetrievalStrategy(final ShardInfo shardInfo) { + return new SynchronousGetRecordsRetrievalStrategy( + new KinesisDataFetcher(amazonKinesis, streamName, shardInfo.shardId(), maxRecords)); + } + + @Override + public GetRecordsCache createGetRecordsCache(final ShardInfo shardInfo) { + return new PrefetchGetRecordsCache(maxPendingProcessRecordsInput, maxByteSize, maxRecordsCount, + maxRecordsPerCall, createGetRecordsRetrievalStrategy(shardInfo), executorService, + idleMillisBetweenCalls, metricsFactory, "Prefetching", shardInfo.shardId()); + } +} diff --git a/amazon-kinesis-client/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/CheckpointImplTestBase.java b/amazon-kinesis-client/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/CheckpointImplTestBase.java index ef0a4ad9..6283380a 100644 --- a/amazon-kinesis-client/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/CheckpointImplTestBase.java +++ b/amazon-kinesis-client/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/CheckpointImplTestBase.java @@ -21,7 +21,7 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -import software.amazon.kinesis.processor.ICheckpoint; +import software.amazon.kinesis.processor.Checkpointer; import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; import software.amazon.kinesis.metrics.MetricsHelper; import software.amazon.kinesis.metrics.NullMetricsFactory; @@ -34,7 +34,7 @@ public abstract class CheckpointImplTestBase { protected final String startingSequenceNumber = "0001000"; protected final String testConcurrencyToken = "testToken"; - protected ICheckpoint checkpoint; + protected Checkpointer checkpoint; /** * @throws java.lang.Exception @@ -107,8 +107,8 @@ public abstract class CheckpointImplTestBase { ExtendedSequenceNumber extendedSequenceNumber = new ExtendedSequenceNumber(checkpointValue); checkpoint.setCheckpoint(shardId, new ExtendedSequenceNumber(checkpointValue), concurrencyToken); Assert.assertEquals(extendedSequenceNumber, checkpoint.getCheckpoint(shardId)); - Assert.assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); - Assert.assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + Assert.assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).checkpoint()); + Assert.assertEquals(null, checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); } @Test @@ -123,8 +123,8 @@ public abstract class CheckpointImplTestBase { checkpoint.prepareCheckpoint(shardId, new ExtendedSequenceNumber(pendingCheckpointValue), testConcurrencyToken); Assert.assertEquals(extendedCheckpointNumber, checkpoint.getCheckpoint(shardId)); - Assert.assertEquals(extendedCheckpointNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); - Assert.assertEquals(extendedPendingCheckpointNumber, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + Assert.assertEquals(extendedCheckpointNumber, checkpoint.getCheckpointObject(shardId).checkpoint()); + Assert.assertEquals(extendedPendingCheckpointNumber, checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); } @Test @@ -139,8 +139,8 @@ public abstract class CheckpointImplTestBase { ExtendedSequenceNumber extendedSequenceNumber = new ExtendedSequenceNumber(sequenceNumber); checkpoint.prepareCheckpoint(shardId, new ExtendedSequenceNumber(sequenceNumber), testConcurrencyToken); Assert.assertEquals(extendedCheckpointNumber, checkpoint.getCheckpoint(shardId)); - Assert.assertEquals(extendedCheckpointNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); - Assert.assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + Assert.assertEquals(extendedCheckpointNumber, checkpoint.getCheckpointObject(shardId).checkpoint()); + Assert.assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); } } @@ -155,20 +155,20 @@ public abstract class CheckpointImplTestBase { ExtendedSequenceNumber extendedCheckpointNumber = new ExtendedSequenceNumber(checkpointValue); checkpoint.setCheckpoint(shardId, new ExtendedSequenceNumber(checkpointValue), concurrencyToken); Assert.assertEquals(extendedCheckpointNumber, checkpoint.getCheckpoint(shardId)); - Assert.assertEquals(extendedCheckpointNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); - Assert.assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + Assert.assertEquals(extendedCheckpointNumber, checkpoint.getCheckpointObject(shardId).checkpoint()); + Assert.assertEquals(null, checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); // prepare checkpoint ExtendedSequenceNumber extendedPendingCheckpointNumber = new ExtendedSequenceNumber(pendingCheckpointValue); checkpoint.prepareCheckpoint(shardId, new ExtendedSequenceNumber(pendingCheckpointValue), concurrencyToken); Assert.assertEquals(extendedCheckpointNumber, checkpoint.getCheckpoint(shardId)); - Assert.assertEquals(extendedCheckpointNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); - Assert.assertEquals(extendedPendingCheckpointNumber, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + Assert.assertEquals(extendedCheckpointNumber, checkpoint.getCheckpointObject(shardId).checkpoint()); + Assert.assertEquals(extendedPendingCheckpointNumber, checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); // do checkpoint checkpoint.setCheckpoint(shardId, new ExtendedSequenceNumber(pendingCheckpointValue), concurrencyToken); Assert.assertEquals(extendedPendingCheckpointNumber, checkpoint.getCheckpoint(shardId)); - Assert.assertEquals(extendedPendingCheckpointNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); - Assert.assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + Assert.assertEquals(extendedPendingCheckpointNumber, checkpoint.getCheckpointObject(shardId).checkpoint()); + Assert.assertEquals(null, checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); } } diff --git a/amazon-kinesis-client/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/InMemoryCheckpointImpl.java b/amazon-kinesis-client/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/InMemoryCheckpointer.java similarity index 96% rename from amazon-kinesis-client/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/InMemoryCheckpointImpl.java rename to amazon-kinesis-client/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/InMemoryCheckpointer.java index ea4219cc..09e512cc 100644 --- a/amazon-kinesis-client/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/InMemoryCheckpointImpl.java +++ b/amazon-kinesis-client/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/InMemoryCheckpointer.java @@ -19,7 +19,7 @@ import java.util.Map; import com.amazonaws.services.kinesis.clientlibrary.exceptions.KinesisClientLibException; import software.amazon.kinesis.checkpoint.Checkpoint; -import software.amazon.kinesis.processor.ICheckpoint; +import software.amazon.kinesis.processor.Checkpointer; import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; import lombok.extern.slf4j.Slf4j; @@ -28,7 +28,7 @@ import lombok.extern.slf4j.Slf4j; * Everything is stored in memory and there is no fault-tolerance. */ @Slf4j -public class InMemoryCheckpointImpl implements ICheckpoint { +public class InMemoryCheckpointer implements Checkpointer { private Map checkpoints = new HashMap<>(); private Map flushpoints = new HashMap<>(); private Map pendingCheckpoints = new HashMap<>(); @@ -39,7 +39,7 @@ public class InMemoryCheckpointImpl implements ICheckpoint { * * @param startingSequenceNumber Initial checkpoint will be set to this sequenceNumber (for all shards). */ - public InMemoryCheckpointImpl(String startingSequenceNumber) { + public InMemoryCheckpointer(String startingSequenceNumber) { super(); this.startingSequenceNumber = startingSequenceNumber; } diff --git a/amazon-kinesis-client/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/InMemoryCheckpointImplTest.java b/amazon-kinesis-client/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/InMemoryCheckpointerTest.java similarity index 82% rename from amazon-kinesis-client/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/InMemoryCheckpointImplTest.java rename to amazon-kinesis-client/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/InMemoryCheckpointerTest.java index 04408b36..d2aec1f8 100644 --- a/amazon-kinesis-client/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/InMemoryCheckpointImplTest.java +++ b/amazon-kinesis-client/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/InMemoryCheckpointerTest.java @@ -20,11 +20,11 @@ import org.junit.Before; /** * Test the InMemoryCheckpointImplTest class. */ -public class InMemoryCheckpointImplTest extends CheckpointImplTestBase { +public class InMemoryCheckpointerTest extends CheckpointImplTestBase { /** * Constructor. */ - public InMemoryCheckpointImplTest() { + public InMemoryCheckpointerTest() { super(); } /** @@ -32,7 +32,7 @@ public class InMemoryCheckpointImplTest extends CheckpointImplTestBase { */ @Before public void setUp() throws Exception { - checkpoint = new InMemoryCheckpointImpl(startingSequenceNumber); + checkpoint = new InMemoryCheckpointer(startingSequenceNumber); } } diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/checkpoint/PreparedCheckpointerTest.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/checkpoint/PreparedCheckpointerTest.java index 99441a6d..ef33b622 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/checkpoint/PreparedCheckpointerTest.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/checkpoint/PreparedCheckpointerTest.java @@ -14,8 +14,6 @@ */ package software.amazon.kinesis.checkpoint; -import software.amazon.kinesis.checkpoint.DoesNothingPreparedCheckpointer; -import software.amazon.kinesis.checkpoint.PreparedCheckpointer; import software.amazon.kinesis.processor.IPreparedCheckpointer; import software.amazon.kinesis.processor.IRecordProcessorCheckpointer; import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; @@ -26,13 +24,13 @@ import org.mockito.Mockito; public class PreparedCheckpointerTest { /** - * This test verifies the relationship between the constructor and getPendingCheckpoint. + * This test verifies the relationship between the constructor and pendingCheckpoint. */ @Test public void testGetSequenceNumber() { ExtendedSequenceNumber sn = new ExtendedSequenceNumber("sn"); IPreparedCheckpointer checkpointer = new PreparedCheckpointer(sn, null); - Assert.assertEquals(sn, checkpointer.getPendingCheckpoint()); + Assert.assertEquals(sn, checkpointer.pendingCheckpoint()); } /** @@ -58,7 +56,7 @@ public class PreparedCheckpointerTest { public void testDoesNothingPreparedCheckpoint() throws Exception { ExtendedSequenceNumber sn = new ExtendedSequenceNumber("sn"); IPreparedCheckpointer checkpointer = new DoesNothingPreparedCheckpointer(sn); - Assert.assertEquals(sn, checkpointer.getPendingCheckpoint()); + Assert.assertEquals(sn, checkpointer.pendingCheckpoint()); // nothing happens here checkpointer.checkpoint(); } diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/checkpoint/RecordProcessorCheckpointerTest.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/checkpoint/RecordProcessorCheckpointerTest.java index b5f9bd28..625edfe3 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/checkpoint/RecordProcessorCheckpointerTest.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/checkpoint/RecordProcessorCheckpointerTest.java @@ -18,8 +18,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import static org.mockito.Matchers.anyString; -import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -36,16 +34,15 @@ import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; -import com.amazonaws.services.kinesis.clientlibrary.lib.checkpoint.InMemoryCheckpointImpl; +import com.amazonaws.services.kinesis.clientlibrary.lib.checkpoint.InMemoryCheckpointer; import com.amazonaws.services.kinesis.model.Record; -import software.amazon.kinesis.coordinator.RecordProcessorCheckpointer; import software.amazon.kinesis.leases.ShardInfo; import software.amazon.kinesis.metrics.IMetricsFactory; import software.amazon.kinesis.metrics.IMetricsScope; import software.amazon.kinesis.metrics.MetricsHelper; import software.amazon.kinesis.metrics.NullMetricsScope; -import software.amazon.kinesis.processor.ICheckpoint; +import software.amazon.kinesis.processor.Checkpointer; import software.amazon.kinesis.processor.IPreparedCheckpointer; import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; import software.amazon.kinesis.retrieval.kpl.UserRecord; @@ -58,7 +55,7 @@ public class RecordProcessorCheckpointerTest { private String startingSequenceNumber = "13"; private ExtendedSequenceNumber startingExtendedSequenceNumber = new ExtendedSequenceNumber(startingSequenceNumber); private String testConcurrencyToken = "testToken"; - private ICheckpoint checkpoint; + private Checkpointer checkpoint; private ShardInfo shardInfo; private String streamName = "testStream"; private String shardId = "shardId-123"; @@ -73,7 +70,7 @@ public class RecordProcessorCheckpointerTest { */ @Before public void setup() throws Exception { - checkpoint = new InMemoryCheckpointImpl(startingSequenceNumber); + checkpoint = new InMemoryCheckpointer(startingSequenceNumber); // A real checkpoint will return a checkpoint value after it is initialized. checkpoint.setCheckpoint(shardId, startingExtendedSequenceNumber, testConcurrencyToken); assertEquals(this.startingExtendedSequenceNumber, checkpoint.getCheckpoint(shardId)); @@ -194,22 +191,22 @@ public class RecordProcessorCheckpointerTest { ExtendedSequenceNumber sequenceNumber1 = new ExtendedSequenceNumber("5001"); processingCheckpointer.largestPermittedCheckpointValue(sequenceNumber1); IPreparedCheckpointer preparedCheckpoint = processingCheckpointer.prepareCheckpoint(); - assertEquals(sequenceNumber1, preparedCheckpoint.getPendingCheckpoint()); - assertEquals(sequenceNumber1, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + assertEquals(sequenceNumber1, preparedCheckpoint.pendingCheckpoint()); + assertEquals(sequenceNumber1, checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); // Advance checkpoint ExtendedSequenceNumber sequenceNumber2 = new ExtendedSequenceNumber("5019"); processingCheckpointer.largestPermittedCheckpointValue(sequenceNumber2); preparedCheckpoint = processingCheckpointer.prepareCheckpoint(); - assertEquals(sequenceNumber2, preparedCheckpoint.getPendingCheckpoint()); - assertEquals(sequenceNumber2, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + assertEquals(sequenceNumber2, preparedCheckpoint.pendingCheckpoint()); + assertEquals(sequenceNumber2, checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); // Checkpoint using preparedCheckpoint preparedCheckpoint.checkpoint(); assertEquals(sequenceNumber2, checkpoint.getCheckpoint(shardId)); - assertEquals(sequenceNumber2, checkpoint.getCheckpointObject(shardId).getCheckpoint()); - assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + assertEquals(sequenceNumber2, checkpoint.getCheckpointObject(shardId).checkpoint()); + assertEquals(null, checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); } /** @@ -226,15 +223,15 @@ public class RecordProcessorCheckpointerTest { processingCheckpointer.largestPermittedCheckpointValue(extendedSequenceNumber); IPreparedCheckpointer preparedCheckpoint = processingCheckpointer.prepareCheckpoint(record); assertEquals(startingExtendedSequenceNumber, checkpoint.getCheckpoint(shardId)); - assertEquals(startingExtendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); - assertEquals(extendedSequenceNumber, preparedCheckpoint.getPendingCheckpoint()); - assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + assertEquals(startingExtendedSequenceNumber, checkpoint.getCheckpointObject(shardId).checkpoint()); + assertEquals(extendedSequenceNumber, preparedCheckpoint.pendingCheckpoint()); + assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); // Checkpoint using preparedCheckpoint preparedCheckpoint.checkpoint(); assertEquals(extendedSequenceNumber, checkpoint.getCheckpoint(shardId)); - assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); - assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).checkpoint()); + assertEquals(null, checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); } /** @@ -252,15 +249,15 @@ public class RecordProcessorCheckpointerTest { processingCheckpointer.largestPermittedCheckpointValue(extendedSequenceNumber); IPreparedCheckpointer preparedCheckpoint = processingCheckpointer.prepareCheckpoint(subRecord); assertEquals(startingExtendedSequenceNumber, checkpoint.getCheckpoint(shardId)); - assertEquals(startingExtendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); - assertEquals(extendedSequenceNumber, preparedCheckpoint.getPendingCheckpoint()); - assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + assertEquals(startingExtendedSequenceNumber, checkpoint.getCheckpointObject(shardId).checkpoint()); + assertEquals(extendedSequenceNumber, preparedCheckpoint.pendingCheckpoint()); + assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); // Checkpoint using preparedCheckpoint preparedCheckpoint.checkpoint(); assertEquals(extendedSequenceNumber, checkpoint.getCheckpoint(shardId)); - assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); - assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).checkpoint()); + assertEquals(null, checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); } /** @@ -276,15 +273,15 @@ public class RecordProcessorCheckpointerTest { processingCheckpointer.largestPermittedCheckpointValue(extendedSequenceNumber); IPreparedCheckpointer preparedCheckpoint = processingCheckpointer.prepareCheckpoint("5035"); assertEquals(startingExtendedSequenceNumber, checkpoint.getCheckpoint(shardId)); - assertEquals(startingExtendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); - assertEquals(extendedSequenceNumber, preparedCheckpoint.getPendingCheckpoint()); - assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + assertEquals(startingExtendedSequenceNumber, checkpoint.getCheckpointObject(shardId).checkpoint()); + assertEquals(extendedSequenceNumber, preparedCheckpoint.pendingCheckpoint()); + assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); // Checkpoint using preparedCheckpoint preparedCheckpoint.checkpoint(); assertEquals(extendedSequenceNumber, checkpoint.getCheckpoint(shardId)); - assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); - assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).checkpoint()); + assertEquals(null, checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); } /** @@ -300,15 +297,15 @@ public class RecordProcessorCheckpointerTest { processingCheckpointer.largestPermittedCheckpointValue(extendedSequenceNumber); IPreparedCheckpointer preparedCheckpoint = processingCheckpointer.prepareCheckpoint("5040", 0); assertEquals(startingExtendedSequenceNumber, checkpoint.getCheckpoint(shardId)); - assertEquals(startingExtendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); - assertEquals(extendedSequenceNumber, preparedCheckpoint.getPendingCheckpoint()); - assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + assertEquals(startingExtendedSequenceNumber, checkpoint.getCheckpointObject(shardId).checkpoint()); + assertEquals(extendedSequenceNumber, preparedCheckpoint.pendingCheckpoint()); + assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); // Checkpoint using preparedCheckpoint preparedCheckpoint.checkpoint(); assertEquals(extendedSequenceNumber, checkpoint.getCheckpoint(shardId)); - assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); - assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).checkpoint()); + assertEquals(null, checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); } /** @@ -323,15 +320,15 @@ public class RecordProcessorCheckpointerTest { processingCheckpointer.largestPermittedCheckpointValue(extendedSequenceNumber); IPreparedCheckpointer preparedCheckpoint = processingCheckpointer.prepareCheckpoint(ExtendedSequenceNumber.SHARD_END.getSequenceNumber()); assertEquals(startingExtendedSequenceNumber, checkpoint.getCheckpoint(shardId)); - assertEquals(startingExtendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); - assertEquals(extendedSequenceNumber, preparedCheckpoint.getPendingCheckpoint()); - assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + assertEquals(startingExtendedSequenceNumber, checkpoint.getCheckpointObject(shardId).checkpoint()); + assertEquals(extendedSequenceNumber, preparedCheckpoint.pendingCheckpoint()); + assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); // Checkpoint using preparedCheckpoint preparedCheckpoint.checkpoint(); assertEquals(extendedSequenceNumber, checkpoint.getCheckpoint(shardId)); - assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); - assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + assertEquals(extendedSequenceNumber, checkpoint.getCheckpointObject(shardId).checkpoint()); + assertEquals(null, checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); } @@ -347,24 +344,24 @@ public class RecordProcessorCheckpointerTest { ExtendedSequenceNumber sn1 = new ExtendedSequenceNumber("6010"); IPreparedCheckpointer firstPreparedCheckpoint = processingCheckpointer.prepareCheckpoint("6010", 0); - assertEquals(sn1, firstPreparedCheckpoint.getPendingCheckpoint()); - assertEquals(sn1, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + assertEquals(sn1, firstPreparedCheckpoint.pendingCheckpoint()); + assertEquals(sn1, checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); ExtendedSequenceNumber sn2 = new ExtendedSequenceNumber("6020"); IPreparedCheckpointer secondPreparedCheckpoint = processingCheckpointer.prepareCheckpoint("6020", 0); - assertEquals(sn2, secondPreparedCheckpoint.getPendingCheckpoint()); - assertEquals(sn2, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + assertEquals(sn2, secondPreparedCheckpoint.pendingCheckpoint()); + assertEquals(sn2, checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); // checkpoint in order firstPreparedCheckpoint.checkpoint(); assertEquals(sn1, checkpoint.getCheckpoint(shardId)); - assertEquals(sn1, checkpoint.getCheckpointObject(shardId).getCheckpoint()); - assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + assertEquals(sn1, checkpoint.getCheckpointObject(shardId).checkpoint()); + assertEquals(null, checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); secondPreparedCheckpoint.checkpoint(); assertEquals(sn2, checkpoint.getCheckpoint(shardId)); - assertEquals(sn2, checkpoint.getCheckpointObject(shardId).getCheckpoint()); - assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + assertEquals(sn2, checkpoint.getCheckpointObject(shardId).checkpoint()); + assertEquals(null, checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); } /** @@ -379,19 +376,19 @@ public class RecordProcessorCheckpointerTest { ExtendedSequenceNumber sn1 = new ExtendedSequenceNumber("7010"); IPreparedCheckpointer firstPreparedCheckpoint = processingCheckpointer.prepareCheckpoint("7010", 0); - assertEquals(sn1, firstPreparedCheckpoint.getPendingCheckpoint()); - assertEquals(sn1, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + assertEquals(sn1, firstPreparedCheckpoint.pendingCheckpoint()); + assertEquals(sn1, checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); ExtendedSequenceNumber sn2 = new ExtendedSequenceNumber("7020"); IPreparedCheckpointer secondPreparedCheckpoint = processingCheckpointer.prepareCheckpoint("7020", 0); - assertEquals(sn2, secondPreparedCheckpoint.getPendingCheckpoint()); - assertEquals(sn2, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + assertEquals(sn2, secondPreparedCheckpoint.pendingCheckpoint()); + assertEquals(sn2, checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); // checkpoint out of order secondPreparedCheckpoint.checkpoint(); assertEquals(sn2, checkpoint.getCheckpoint(shardId)); - assertEquals(sn2, checkpoint.getCheckpointObject(shardId).getCheckpoint()); - assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + assertEquals(sn2, checkpoint.getCheckpointObject(shardId).checkpoint()); + assertEquals(null, checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); try { firstPreparedCheckpoint.checkpoint(); @@ -549,23 +546,23 @@ public class RecordProcessorCheckpointerTest { IPreparedCheckpointer doesNothingPreparedCheckpoint = processingCheckpointer.prepareCheckpoint(firstSequenceNumber.getSequenceNumber(), firstSequenceNumber.getSubSequenceNumber()); assertTrue(doesNothingPreparedCheckpoint instanceof DoesNothingPreparedCheckpointer); - assertEquals(firstSequenceNumber, doesNothingPreparedCheckpoint.getPendingCheckpoint()); + assertEquals(firstSequenceNumber, doesNothingPreparedCheckpoint.pendingCheckpoint()); assertEquals(firstSequenceNumber, checkpoint.getCheckpoint(shardId)); - assertEquals(firstSequenceNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); - assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + assertEquals(firstSequenceNumber, checkpoint.getCheckpointObject(shardId).checkpoint()); + assertEquals(null, checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); // nothing happens after checkpointing a doesNothingPreparedCheckpoint doesNothingPreparedCheckpoint.checkpoint(); assertEquals(firstSequenceNumber, checkpoint.getCheckpoint(shardId)); - assertEquals(firstSequenceNumber, checkpoint.getCheckpointObject(shardId).getCheckpoint()); - assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + assertEquals(firstSequenceNumber, checkpoint.getCheckpointObject(shardId).checkpoint()); + assertEquals(null, checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); // advance to second processingCheckpointer.prepareCheckpoint(secondSequenceNumber.getSequenceNumber(), secondSequenceNumber.getSubSequenceNumber()); - assertEquals(secondSequenceNumber, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + assertEquals(secondSequenceNumber, checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); processingCheckpointer.checkpoint(secondSequenceNumber.getSequenceNumber(), secondSequenceNumber.getSubSequenceNumber()); assertEquals(secondSequenceNumber, checkpoint.getCheckpoint(shardId)); - assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + assertEquals(null, checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); ExtendedSequenceNumber[] valuesWeShouldNotBeAbleToCheckpointAt = { tooSmall, // Shouldn't be able to move before the first value we ever checkpointed @@ -596,13 +593,13 @@ public class RecordProcessorCheckpointerTest { assertEquals("Largest sequence number should not have changed", thirdSequenceNumber, processingCheckpointer.largestPermittedCheckpointValue()); - assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + assertEquals(null, checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); } // advance to third number processingCheckpointer.prepareCheckpoint(thirdSequenceNumber.getSequenceNumber(), thirdSequenceNumber.getSubSequenceNumber()); - assertEquals(thirdSequenceNumber, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + assertEquals(thirdSequenceNumber, checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); processingCheckpointer.checkpoint(thirdSequenceNumber.getSequenceNumber(), thirdSequenceNumber.getSubSequenceNumber()); assertEquals(thirdSequenceNumber, checkpoint.getCheckpoint(shardId)); @@ -614,7 +611,7 @@ public class RecordProcessorCheckpointerTest { assertEquals("Preparing a checkpoing at the sequence number at the end of a shard should be the same as " + "preparing a checkpoint at SHARD_END", ExtendedSequenceNumber.SHARD_END, - checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); } private enum CheckpointAction { @@ -788,8 +785,8 @@ public class RecordProcessorCheckpointerTest { case PREPARE_THEN_CHECKPOINTER: preparedCheckpoint = processingCheckpointer.prepareCheckpoint(); processingCheckpointer.checkpoint( - preparedCheckpoint.getPendingCheckpoint().getSequenceNumber(), - preparedCheckpoint.getPendingCheckpoint().getSubSequenceNumber()); + preparedCheckpoint.pendingCheckpoint().getSequenceNumber(), + preparedCheckpoint.pendingCheckpoint().getSubSequenceNumber()); } break; case WITH_SEQUENCE_NUMBER: @@ -803,8 +800,8 @@ public class RecordProcessorCheckpointerTest { case PREPARE_THEN_CHECKPOINTER: preparedCheckpoint = processingCheckpointer.prepareCheckpoint(entry.getKey()); processingCheckpointer.checkpoint( - preparedCheckpoint.getPendingCheckpoint().getSequenceNumber(), - preparedCheckpoint.getPendingCheckpoint().getSubSequenceNumber()); + preparedCheckpoint.pendingCheckpoint().getSequenceNumber(), + preparedCheckpoint.pendingCheckpoint().getSubSequenceNumber()); } break; } @@ -818,8 +815,8 @@ public class RecordProcessorCheckpointerTest { assertEquals(new ExtendedSequenceNumber(entry.getKey()), checkpoint.getCheckpoint(shardId)); assertEquals(new ExtendedSequenceNumber(entry.getKey()), - checkpoint.getCheckpointObject(shardId).getCheckpoint()); - assertEquals(null, checkpoint.getCheckpointObject(shardId).getPendingCheckpoint()); + checkpoint.getCheckpointObject(shardId).checkpoint()); + assertEquals(null, checkpoint.getCheckpointObject(shardId).pendingCheckpoint()); } } diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/coordinator/KinesisClientLibConfigurationTest.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/coordinator/KinesisClientLibConfigurationTest.java index 5e1cb436..f7733fd0 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/coordinator/KinesisClientLibConfigurationTest.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/coordinator/KinesisClientLibConfigurationTest.java @@ -20,26 +20,6 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertFalse; import static org.junit.Assert.fail; -import java.util.Date; - -import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStream; -import org.junit.Test; -import org.mockito.Mockito; - -import com.amazonaws.ClientConfiguration; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.regions.Region; -import com.amazonaws.regions.RegionUtils; -import com.amazonaws.services.cloudwatch.AmazonCloudWatchClient; -import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; -import com.amazonaws.services.kinesis.AmazonKinesisClient; -import software.amazon.kinesis.coordinator.KinesisClientLibConfiguration; -import software.amazon.kinesis.processor.IRecordProcessorFactory; -import software.amazon.kinesis.metrics.MetricsLevel; -import com.google.common.collect.ImmutableSet; - -import junit.framework.Assert; - public class KinesisClientLibConfigurationTest { /*private static final long INVALID_LONG = 0L; private static final int INVALID_INT = 0; diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/coordinator/SchedulerTest.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/coordinator/SchedulerTest.java index dbed7d6c..0d2e4b1d 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/coordinator/SchedulerTest.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/coordinator/SchedulerTest.java @@ -67,8 +67,8 @@ import software.amazon.kinesis.lifecycle.ShutdownInput; import software.amazon.kinesis.lifecycle.ShutdownReason; import software.amazon.kinesis.metrics.IMetricsFactory; import software.amazon.kinesis.metrics.MetricsConfig; -import software.amazon.kinesis.processor.ICheckpoint; -import software.amazon.kinesis.processor.IRecordProcessor; +import software.amazon.kinesis.processor.Checkpointer; +import software.amazon.kinesis.processor.RecordProcessor; import software.amazon.kinesis.processor.ProcessorConfig; import software.amazon.kinesis.processor.ProcessorFactory; import software.amazon.kinesis.retrieval.GetRecordsCache; @@ -114,7 +114,7 @@ public class SchedulerTest { @Mock private LeaseManagerProxy leaseManagerProxy; @Mock - private ICheckpoint checkpoint; + private Checkpointer checkpoint; @Before public void setup() { @@ -399,8 +399,8 @@ public class SchedulerTest { private static class TestRecordProcessorFactory implements ProcessorFactory { @Override - public IRecordProcessor createRecordProcessor() { - return new IRecordProcessor() { + public RecordProcessor createRecordProcessor() { + return new RecordProcessor() { @Override public void initialize(final InitializationInput initializationInput) { // Do nothing. @@ -458,7 +458,7 @@ public class SchedulerTest { private class TestKinesisCheckpointFactory implements CheckpointFactory { @Override - public ICheckpoint createCheckpoint(KinesisClientLibLeaseCoordinator leaseCoordinator) { + public Checkpointer createCheckpoint() { return checkpoint; } } diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/LeaseRenewerIntegrationTest.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/DynamoDBLeaseRenewerIntegrationTest.java similarity index 100% rename from amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/LeaseRenewerIntegrationTest.java rename to amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/DynamoDBLeaseRenewerIntegrationTest.java diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/LeaseRenewerTest.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/DynamoDBLeaseRenewerTest.java similarity index 100% rename from amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/LeaseRenewerTest.java rename to amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/DynamoDBLeaseRenewerTest.java diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/LeaseTakerIntegrationTest.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/DynamoDBLeaseTakerIntegrationTest.java similarity index 95% rename from amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/LeaseTakerIntegrationTest.java rename to amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/DynamoDBLeaseTakerIntegrationTest.java index 70aed06f..de7319fc 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/LeaseTakerIntegrationTest.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/DynamoDBLeaseTakerIntegrationTest.java @@ -22,14 +22,14 @@ import org.junit.Test; import software.amazon.kinesis.leases.exceptions.LeasingException; -public class LeaseTakerIntegrationTest extends LeaseIntegrationTest { +public class DynamoDBLeaseTakerIntegrationTest extends LeaseIntegrationTest { private static final long LEASE_DURATION_MILLIS = 1000L; - private LeaseTaker taker; + private DynamoDBLeaseTaker taker; @Before public void setUp() { - taker = new LeaseTaker(leaseManager, "foo", LEASE_DURATION_MILLIS); + taker = new DynamoDBLeaseTaker(leaseManager, "foo", LEASE_DURATION_MILLIS); } @Test diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/LeaseTakerTest.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/DynamoDBLeaseTakerTest.java similarity index 82% rename from amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/LeaseTakerTest.java rename to amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/DynamoDBLeaseTakerTest.java index a0e41e23..a4b91c6e 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/LeaseTakerTest.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/DynamoDBLeaseTakerTest.java @@ -24,12 +24,11 @@ import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -import software.amazon.kinesis.leases.LeaseTaker; /** * */ -public class LeaseTakerTest { +public class DynamoDBLeaseTakerTest { /** * @throws java.lang.Exception @@ -60,17 +59,17 @@ public class LeaseTakerTest { } /** - * Test method for {@link LeaseTaker#stringJoin(java.util.Collection, java.lang.String)}. + * Test method for {@link DynamoDBLeaseTaker#stringJoin(java.util.Collection, java.lang.String)}. */ @Test public final void testStringJoin() { List strings = new ArrayList<>(); strings.add("foo"); - Assert.assertEquals("foo", LeaseTaker.stringJoin(strings, ", ")); + Assert.assertEquals("foo", DynamoDBLeaseTaker.stringJoin(strings, ", ")); strings.add("bar"); - Assert.assertEquals("foo, bar", LeaseTaker.stringJoin(strings, ", ")); + Assert.assertEquals("foo, bar", DynamoDBLeaseTaker.stringJoin(strings, ", ")); } } diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/KinesisClientLibLeaseCoordinatorIntegrationTest.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/KinesisClientLibLeaseCoordinatorIntegrationTest.java index 889b1932..b54d6d30 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/KinesisClientLibLeaseCoordinatorIntegrationTest.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/KinesisClientLibLeaseCoordinatorIntegrationTest.java @@ -14,6 +14,11 @@ */ package software.amazon.kinesis.leases; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -22,31 +27,38 @@ import java.util.Map; import java.util.UUID; import java.util.concurrent.Callable; -import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; -import junit.framework.Assert; - import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.runners.MockitoJUnitRunner; - +import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; -import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; + +import software.amazon.kinesis.checkpoint.DynamoDBCheckpointer; import software.amazon.kinesis.leases.exceptions.DependencyException; import software.amazon.kinesis.leases.exceptions.InvalidStateException; import software.amazon.kinesis.leases.exceptions.LeasingException; import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException; +import software.amazon.kinesis.metrics.IMetricsFactory; +import software.amazon.kinesis.metrics.NullMetricsFactory; +import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; -import static org.hamcrest.CoreMatchers.notNullValue; -import static org.hamcrest.MatcherAssert.assertThat; - +@RunWith(MockitoJUnitRunner.class) public class KinesisClientLibLeaseCoordinatorIntegrationTest { - - private static KinesisClientLeaseManager leaseManager; - private KinesisClientLibLeaseCoordinator coordinator; private static final String TABLE_NAME = KinesisClientLibLeaseCoordinatorIntegrationTest.class.getSimpleName(); private static final String WORKER_ID = UUID.randomUUID().toString(); - private final String leaseKey = "shd-1"; + private static final long LEASE_DURATION_MILLIS = 5000L; + private static final long EPSILON_MILLIS = 25L; + private static final int MAX_LEASES_FOR_WORKER = Integer.MAX_VALUE; + private static final int MAX_LEASES_TO_STEAL_AT_ONE_TIME = 1; + private static final int MAX_LEASE_RENEWER_THREAD_COUNT = 20; + private static KinesisClientLeaseManager leaseManager; + private static DynamoDBCheckpointer dynamoDBCheckpointer; + private KinesisClientLibLeaseCoordinator coordinator; + private final String leaseKey = "shd-1"; + private final IMetricsFactory metricsFactory = new NullMetricsFactory(); @Before public void setUp() throws ProvisionedThroughputException, DependencyException, InvalidStateException { @@ -58,7 +70,11 @@ public class KinesisClientLibLeaseCoordinatorIntegrationTest { } leaseManager.createLeaseTableIfNotExists(10L, 10L); leaseManager.deleteAll(); - coordinator = new KinesisClientLibLeaseCoordinator(leaseManager, WORKER_ID, 5000L, 50L); + coordinator = new KinesisClientLibLeaseCoordinator(leaseManager, WORKER_ID, LEASE_DURATION_MILLIS, + EPSILON_MILLIS, MAX_LEASES_FOR_WORKER, MAX_LEASES_TO_STEAL_AT_ONE_TIME, MAX_LEASE_RENEWER_THREAD_COUNT, + metricsFactory); + dynamoDBCheckpointer = new DynamoDBCheckpointer(coordinator, leaseManager, metricsFactory); + coordinator.start(); } @@ -66,7 +82,7 @@ public class KinesisClientLibLeaseCoordinatorIntegrationTest { * Tests update checkpoint success. */ @Test - public void testUpdateCheckpoint() throws LeasingException { + public void testUpdateCheckpoint() throws Exception { TestHarnessBuilder builder = new TestHarnessBuilder(); builder.withLease(leaseKey, null).build(); @@ -82,17 +98,17 @@ public class KinesisClientLibLeaseCoordinatorIntegrationTest { } } - assertThat(lease, notNullValue()); + assertNotNull(lease); ExtendedSequenceNumber newCheckpoint = new ExtendedSequenceNumber("newCheckpoint"); // lease's leaseCounter is wrong at this point, but it shouldn't matter. - Assert.assertTrue(coordinator.setCheckpoint(lease.getLeaseKey(), newCheckpoint, lease.getConcurrencyToken())); + assertTrue(dynamoDBCheckpointer.setCheckpoint(lease.getLeaseKey(), newCheckpoint, lease.getConcurrencyToken())); Lease fromDynamo = leaseManager.getLease(lease.getLeaseKey()); lease.setLeaseCounter(lease.getLeaseCounter() + 1); lease.setCheckpoint(newCheckpoint); lease.setLeaseOwner(coordinator.getWorkerIdentifier()); - Assert.assertEquals(lease, fromDynamo); + assertEquals(lease, fromDynamo); } /** @@ -107,18 +123,18 @@ public class KinesisClientLibLeaseCoordinatorIntegrationTest { coordinator.runLeaseRenewer(); KinesisClientLease lease = coordinator.getCurrentlyHeldLease(leaseKey); - assertThat(lease, notNullValue()); + assertNotNull(lease); leaseManager.renewLease(coordinator.getCurrentlyHeldLease(leaseKey)); ExtendedSequenceNumber newCheckpoint = new ExtendedSequenceNumber("newCheckpoint"); - Assert.assertFalse(coordinator.setCheckpoint(lease.getLeaseKey(), newCheckpoint, lease.getConcurrencyToken())); + assertFalse(dynamoDBCheckpointer.setCheckpoint(lease.getLeaseKey(), newCheckpoint, lease.getConcurrencyToken())); Lease fromDynamo = leaseManager.getLease(lease.getLeaseKey()); lease.setLeaseCounter(lease.getLeaseCounter() + 1); // Counter and owner changed, but checkpoint did not. lease.setLeaseOwner(coordinator.getWorkerIdentifier()); - Assert.assertEquals(lease, fromDynamo); + assertEquals(lease, fromDynamo); } /** @@ -133,16 +149,16 @@ public class KinesisClientLibLeaseCoordinatorIntegrationTest { coordinator.runLeaseRenewer(); KinesisClientLease lease = coordinator.getCurrentlyHeldLease(leaseKey); - assertThat(lease, notNullValue()); + assertNotNull(lease); ExtendedSequenceNumber newCheckpoint = new ExtendedSequenceNumber("newCheckpoint"); - Assert.assertFalse(coordinator.setCheckpoint(lease.getLeaseKey(), newCheckpoint, UUID.randomUUID())); + assertFalse(dynamoDBCheckpointer.setCheckpoint(lease.getLeaseKey(), newCheckpoint, UUID.randomUUID())); Lease fromDynamo = leaseManager.getLease(lease.getLeaseKey()); // Owner should be the only thing that changed. lease.setLeaseOwner(coordinator.getWorkerIdentifier()); - Assert.assertEquals(lease, fromDynamo); + assertEquals(lease, fromDynamo); } public static class TestHarnessBuilder { @@ -201,7 +217,7 @@ public class KinesisClientLibLeaseCoordinatorIntegrationTest { } original.setLeaseOwner(newWorkerIdentifier); - Assert.assertEquals(original, actual); // Assert the contents of the lease + assertEquals(original, actual); // Assert the contents of the lease } public void addLeasesToRenew(ILeaseRenewer renewer, String... shardIds) @@ -210,7 +226,7 @@ public class KinesisClientLibLeaseCoordinatorIntegrationTest { for (String shardId : shardIds) { KinesisClientLease lease = leases.get(shardId); - Assert.assertNotNull(lease); + assertNotNull(lease); leasesToRenew.add(lease); } @@ -222,17 +238,17 @@ public class KinesisClientLibLeaseCoordinatorIntegrationTest { renewer.renewLeases(); Map heldLeases = renewer.getCurrentlyHeldLeases(); - Assert.assertEquals(renewedShardIds.length, heldLeases.size()); + assertEquals(renewedShardIds.length, heldLeases.size()); for (String shardId : renewedShardIds) { KinesisClientLease original = leases.get(shardId); - Assert.assertNotNull(original); + assertNotNull(original); KinesisClientLease actual = heldLeases.get(shardId); - Assert.assertNotNull(actual); + assertNotNull(actual); original.setLeaseCounter(original.getLeaseCounter() + 1); - Assert.assertEquals(original, actual); + assertEquals(original, actual); } return heldLeases; diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/KinesisClientLibLeaseCoordinatorTest.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/KinesisClientLibLeaseCoordinatorTest.java index 5a231f80..f7d81f5f 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/KinesisClientLibLeaseCoordinatorTest.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/KinesisClientLibLeaseCoordinatorTest.java @@ -14,27 +14,28 @@ */ package software.amazon.kinesis.leases; -import static org.mockito.Matchers.anyLong; -import static org.mockito.Mockito.doReturn; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.util.UUID; -import junit.framework.Assert; - import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; import org.mockito.Mock; -import org.mockito.MockitoAnnotations; import com.amazonaws.services.kinesis.clientlibrary.exceptions.KinesisClientLibException; import com.amazonaws.services.kinesis.clientlibrary.exceptions.ShutdownException; -import software.amazon.kinesis.leases.KinesisClientLibLeaseCoordinator; +import org.mockito.runners.MockitoJUnitRunner; +import software.amazon.kinesis.checkpoint.DynamoDBCheckpointer; +import software.amazon.kinesis.metrics.IMetricsFactory; import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; import software.amazon.kinesis.leases.exceptions.DependencyException; import software.amazon.kinesis.leases.exceptions.InvalidStateException; import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException; -import software.amazon.kinesis.leases.ILeaseManager; +@RunWith(MockitoJUnitRunner.class) public class KinesisClientLibLeaseCoordinatorTest { private static final String SHARD_ID = "shardId-test"; private static final String WORK_ID = "workId-test"; @@ -42,35 +43,38 @@ public class KinesisClientLibLeaseCoordinatorTest { private static final ExtendedSequenceNumber TEST_CHKPT = new ExtendedSequenceNumber("string-test"); private static final UUID TEST_UUID = UUID.randomUUID(); - @SuppressWarnings("rawtypes") @Mock - private ILeaseManager mockLeaseManager; + private ILeaseManager leaseManager; + @Mock + private LeaseCoordinator leaseCoordinator; + @Mock + private IMetricsFactory metricsFactory; - private KinesisClientLibLeaseCoordinator leaseCoordinator; + private DynamoDBCheckpointer dynamoDBCheckpointer; @SuppressWarnings("unchecked") @Before public void setUpLeaseCoordinator() throws ProvisionedThroughputException, DependencyException { - // Initialize the annotation - MockitoAnnotations.initMocks(this); - // Set up lease coordinator - doReturn(true).when(mockLeaseManager).createLeaseTableIfNotExists(anyLong(), anyLong()); - leaseCoordinator = new KinesisClientLibLeaseCoordinator(mockLeaseManager, WORK_ID, TEST_LONG, TEST_LONG); + dynamoDBCheckpointer = new DynamoDBCheckpointer(leaseCoordinator, leaseManager, metricsFactory); } @Test(expected = ShutdownException.class) - public void testSetCheckpointWithUnownedShardId() - throws KinesisClientLibException, DependencyException, InvalidStateException, ProvisionedThroughputException { - final boolean succeess = leaseCoordinator.setCheckpoint(SHARD_ID, TEST_CHKPT, TEST_UUID); - Assert.assertFalse("Set Checkpoint should return failure", succeess); - leaseCoordinator.setCheckpoint(SHARD_ID, TEST_CHKPT, TEST_UUID.toString()); + public void testSetCheckpointWithUnownedShardId() throws KinesisClientLibException, DependencyException, InvalidStateException, ProvisionedThroughputException { + final KinesisClientLease lease = new KinesisClientLease(); + when(leaseManager.getLease(eq(SHARD_ID))).thenReturn(lease); + when(leaseCoordinator.updateLease(eq(lease), eq(TEST_UUID))).thenReturn(false); + + dynamoDBCheckpointer.setCheckpoint(SHARD_ID, TEST_CHKPT, TEST_UUID.toString()); + + verify(leaseManager).getLease(eq(SHARD_ID)); + verify(leaseCoordinator).updateLease(eq(lease), eq(TEST_UUID)); } - @Test(expected = DependencyException.class) - public void testWaitLeaseTableTimeout() - throws DependencyException, ProvisionedThroughputException, IllegalStateException { - // Set mock lease manager to return false in waiting - doReturn(false).when(mockLeaseManager).waitUntilLeaseTableExists(anyLong(), anyLong()); - leaseCoordinator.initialize(); - } +// @Test(expected = DependencyException.class) +// public void testWaitLeaseTableTimeout() +// throws DependencyException, ProvisionedThroughputException, IllegalStateException { +// Set mock lease manager to return false in waiting +// doReturn(false).when(leaseManager).waitUntilLeaseTableExists(anyLong(), anyLong()); +// leaseCoordinator.initialize(); +// } } diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/TestHarnessBuilder.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/TestHarnessBuilder.java index 104ba221..fe56f509 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/TestHarnessBuilder.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/TestHarnessBuilder.java @@ -91,7 +91,7 @@ public class TestHarnessBuilder { currentTimeNanos += millis * 1000000; } - public Map takeMutateAssert(LeaseTaker taker, int numToTake) + public Map takeMutateAssert(DynamoDBLeaseTaker taker, int numToTake) throws LeasingException { Map result = taker.takeLeases(timeProvider); Assert.assertEquals(numToTake, result.size()); @@ -106,7 +106,7 @@ public class TestHarnessBuilder { return result; } - public Map takeMutateAssert(LeaseTaker taker, String... takenShardIds) + public Map takeMutateAssert(DynamoDBLeaseTaker taker, String... takenShardIds) throws LeasingException { Map result = taker.takeLeases(timeProvider); Assert.assertEquals(takenShardIds.length, result.size()); diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/ConsumerStatesTest.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/ConsumerStatesTest.java index b94aad6c..286daae6 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/ConsumerStatesTest.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/ConsumerStatesTest.java @@ -44,14 +44,14 @@ import com.amazonaws.services.kinesis.AmazonKinesis; import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStream; import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStreamExtended; -import software.amazon.kinesis.coordinator.RecordProcessorCheckpointer; +import software.amazon.kinesis.checkpoint.RecordProcessorCheckpointer; import software.amazon.kinesis.leases.ILeaseManager; import software.amazon.kinesis.leases.KinesisClientLease; import software.amazon.kinesis.leases.LeaseManagerProxy; import software.amazon.kinesis.leases.ShardInfo; import software.amazon.kinesis.metrics.IMetricsFactory; -import software.amazon.kinesis.processor.ICheckpoint; -import software.amazon.kinesis.processor.IRecordProcessor; +import software.amazon.kinesis.processor.Checkpointer; +import software.amazon.kinesis.processor.RecordProcessor; import software.amazon.kinesis.processor.IRecordProcessorCheckpointer; import software.amazon.kinesis.retrieval.GetRecordsCache; @@ -64,7 +64,7 @@ public class ConsumerStatesTest { private ShardConsumer consumer; @Mock - private IRecordProcessor recordProcessor; + private RecordProcessor recordProcessor; @Mock private RecordProcessorCheckpointer recordProcessorCheckpointer; @Mock @@ -74,7 +74,7 @@ public class ConsumerStatesTest { @Mock private ILeaseManager leaseManager; @Mock - private ICheckpoint checkpoint; + private Checkpointer checkpoint; @Mock private ShutdownNotification shutdownNotification; @Mock @@ -144,8 +144,8 @@ public class ConsumerStatesTest { ITask task = state.createTask(consumer); assertThat(task, initTask(ShardInfo.class, "shardInfo", equalTo(shardInfo))); - assertThat(task, initTask(IRecordProcessor.class, "recordProcessor", equalTo(recordProcessor))); - assertThat(task, initTask(ICheckpoint.class, "checkpoint", equalTo(checkpoint))); + assertThat(task, initTask(RecordProcessor.class, "recordProcessor", equalTo(recordProcessor))); + assertThat(task, initTask(Checkpointer.class, "checkpoint", equalTo(checkpoint))); assertThat(task, initTask(RecordProcessorCheckpointer.class, "recordProcessorCheckpointer", equalTo(recordProcessorCheckpointer))); assertThat(task, initTask(Long.class, "backoffTimeMillis", equalTo(taskBackoffTimeMillis))); @@ -171,7 +171,7 @@ public class ConsumerStatesTest { ITask task = state.createTask(consumer); assertThat(task, procTask(ShardInfo.class, "shardInfo", equalTo(shardInfo))); - assertThat(task, procTask(IRecordProcessor.class, "recordProcessor", equalTo(recordProcessor))); + assertThat(task, procTask(RecordProcessor.class, "recordProcessor", equalTo(recordProcessor))); assertThat(task, procTask(RecordProcessorCheckpointer.class, "recordProcessorCheckpointer", equalTo(recordProcessorCheckpointer))); assertThat(task, procTask(Long.class, "backoffTimeMillis", equalTo(taskBackoffTimeMillis))); @@ -198,7 +198,7 @@ public class ConsumerStatesTest { ITask task = state.createTask(consumer); assertThat(task, procTask(ShardInfo.class, "shardInfo", equalTo(shardInfo))); - assertThat(task, procTask(IRecordProcessor.class, "recordProcessor", equalTo(recordProcessor))); + assertThat(task, procTask(RecordProcessor.class, "recordProcessor", equalTo(recordProcessor))); assertThat(task, procTask(RecordProcessorCheckpointer.class, "recordProcessorCheckpointer", equalTo(recordProcessorCheckpointer))); assertThat(task, procTask(Long.class, "backoffTimeMillis", equalTo(taskBackoffTimeMillis))); @@ -225,7 +225,7 @@ public class ConsumerStatesTest { ITask task = state.createTask(consumer); assertThat(task, procTask(ShardInfo.class, "shardInfo", equalTo(shardInfo))); - assertThat(task, procTask(IRecordProcessor.class, "recordProcessor", equalTo(recordProcessor))); + assertThat(task, procTask(RecordProcessor.class, "recordProcessor", equalTo(recordProcessor))); assertThat(task, procTask(RecordProcessorCheckpointer.class, "recordProcessorCheckpointer", equalTo(recordProcessorCheckpointer))); assertThat(task, procTask(Long.class, "backoffTimeMillis", equalTo(taskBackoffTimeMillis))); @@ -250,7 +250,7 @@ public class ConsumerStatesTest { consumer.notifyShutdownRequested(shutdownNotification); ITask task = state.createTask(consumer); - assertThat(task, shutdownReqTask(IRecordProcessor.class, "recordProcessor", equalTo(recordProcessor))); + assertThat(task, shutdownReqTask(RecordProcessor.class, "recordProcessor", equalTo(recordProcessor))); assertThat(task, shutdownReqTask(IRecordProcessorCheckpointer.class, "recordProcessorCheckpointer", equalTo(recordProcessorCheckpointer))); assertThat(task, shutdownReqTask(ShutdownNotification.class, "shutdownNotification", equalTo(shutdownNotification))); @@ -296,7 +296,7 @@ public class ConsumerStatesTest { ITask task = state.createTask(consumer); assertThat(task, shutdownTask(ShardInfo.class, "shardInfo", equalTo(shardInfo))); - assertThat(task, shutdownTask(IRecordProcessor.class, "recordProcessor", equalTo(recordProcessor))); + assertThat(task, shutdownTask(RecordProcessor.class, "recordProcessor", equalTo(recordProcessor))); assertThat(task, shutdownTask(RecordProcessorCheckpointer.class, "recordProcessorCheckpointer", equalTo(recordProcessorCheckpointer))); assertThat(task, shutdownTask(ShutdownReason.class, "reason", equalTo(reason))); diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/ProcessTaskTest.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/ProcessTaskTest.java index 6b11e55d..807adcfc 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/ProcessTaskTest.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/ProcessTaskTest.java @@ -46,10 +46,10 @@ import com.amazonaws.services.kinesis.model.Record; import com.google.protobuf.ByteString; import lombok.Data; -import software.amazon.kinesis.coordinator.RecordProcessorCheckpointer; +import software.amazon.kinesis.checkpoint.RecordProcessorCheckpointer; import software.amazon.kinesis.leases.LeaseManagerProxy; import software.amazon.kinesis.leases.ShardInfo; -import software.amazon.kinesis.processor.IRecordProcessor; +import software.amazon.kinesis.processor.RecordProcessor; import software.amazon.kinesis.retrieval.ThrottlingReporter; import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; import software.amazon.kinesis.retrieval.kpl.Messages; @@ -79,7 +79,7 @@ public class ProcessTaskTest { private final long taskBackoffTimeMillis = 1L; @Mock - private IRecordProcessor recordProcessor; + private RecordProcessor recordProcessor; @Mock private RecordProcessorCheckpointer checkpointer; @Mock diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/ShardConsumerTest.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/ShardConsumerTest.java index 3a7a9c3b..561657f6 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/ShardConsumerTest.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/ShardConsumerTest.java @@ -62,7 +62,7 @@ import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; import com.amazonaws.services.kinesis.AmazonKinesis; -import com.amazonaws.services.kinesis.clientlibrary.lib.checkpoint.InMemoryCheckpointImpl; +import com.amazonaws.services.kinesis.clientlibrary.lib.checkpoint.InMemoryCheckpointer; import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStream; import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStreamExtended; import com.amazonaws.services.kinesis.clientlibrary.proxies.KinesisLocalFileProxy; @@ -74,15 +74,15 @@ import com.amazonaws.services.kinesis.model.ShardIteratorType; import lombok.extern.slf4j.Slf4j; import software.amazon.kinesis.checkpoint.Checkpoint; import software.amazon.kinesis.coordinator.KinesisClientLibConfiguration; -import software.amazon.kinesis.coordinator.RecordProcessorCheckpointer; +import software.amazon.kinesis.checkpoint.RecordProcessorCheckpointer; import software.amazon.kinesis.leases.ILeaseManager; import software.amazon.kinesis.leases.KinesisClientLease; import software.amazon.kinesis.leases.LeaseManagerProxy; import software.amazon.kinesis.leases.ShardInfo; import software.amazon.kinesis.metrics.IMetricsFactory; import software.amazon.kinesis.metrics.NullMetricsFactory; -import software.amazon.kinesis.processor.ICheckpoint; -import software.amazon.kinesis.processor.IRecordProcessor; +import software.amazon.kinesis.processor.Checkpointer; +import software.amazon.kinesis.processor.RecordProcessor; import software.amazon.kinesis.retrieval.AsynchronousGetRecordsRetrievalStrategy; import software.amazon.kinesis.retrieval.BlockingGetRecordsCache; import software.amazon.kinesis.retrieval.GetRecordsCache; @@ -135,13 +135,13 @@ public class ShardConsumerTest { @Mock private RecordsFetcherFactory recordsFetcherFactory; @Mock - private IRecordProcessor recordProcessor; + private RecordProcessor recordProcessor; @Mock private KinesisClientLibConfiguration config; @Mock private ILeaseManager leaseManager; @Mock - private ICheckpoint checkpoint; + private Checkpointer checkpoint; @Mock private ShutdownNotification shutdownNotification; @Mock @@ -287,7 +287,7 @@ public class ShardConsumerTest { IKinesisProxy fileBasedProxy = new KinesisLocalFileProxy(file.getAbsolutePath()); final int maxRecords = 2; - ICheckpoint checkpoint = new InMemoryCheckpointImpl(startSeqNum.toString()); + Checkpointer checkpoint = new InMemoryCheckpointer(startSeqNum.toString()); checkpoint.setCheckpoint(shardId, ExtendedSequenceNumber.TRIM_HORIZON, concurrencyToken); when(leaseManager.getLease(anyString())).thenReturn(null); TestStreamlet processor = new TestStreamlet(); @@ -390,7 +390,7 @@ public class ShardConsumerTest { IKinesisProxy fileBasedProxy = new KinesisLocalFileProxy(file.getAbsolutePath()); final int maxRecords = 2; - ICheckpoint checkpoint = new InMemoryCheckpointImpl(startSeqNum.toString()); + Checkpointer checkpoint = new InMemoryCheckpointer(startSeqNum.toString()); checkpoint.setCheckpoint(shardId, ExtendedSequenceNumber.TRIM_HORIZON, concurrencyToken); when(leaseManager.getLease(anyString())).thenReturn(null); @@ -485,7 +485,7 @@ public class ShardConsumerTest { IKinesisProxy fileBasedProxy = new KinesisLocalFileProxy(file.getAbsolutePath()); final int maxRecords = 2; - ICheckpoint checkpoint = new InMemoryCheckpointImpl(startSeqNum.toString()); + Checkpointer checkpoint = new InMemoryCheckpointer(startSeqNum.toString()); checkpoint.setCheckpoint(streamShardId, ExtendedSequenceNumber.AT_TIMESTAMP, testConcurrencyToken); when(leaseManager.getLease(anyString())).thenReturn(null); TestStreamlet processor = new TestStreamlet(); diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/ShutdownTaskTest.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/ShutdownTaskTest.java index 4ba6e067..d552564c 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/ShutdownTaskTest.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/ShutdownTaskTest.java @@ -33,12 +33,12 @@ import com.amazonaws.services.kinesis.clientlibrary.exceptions.internal.KinesisC import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStream; import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStreamExtended; -import software.amazon.kinesis.coordinator.RecordProcessorCheckpointer; +import software.amazon.kinesis.checkpoint.RecordProcessorCheckpointer; import software.amazon.kinesis.leases.ILeaseManager; import software.amazon.kinesis.leases.KinesisClientLease; import software.amazon.kinesis.leases.LeaseManagerProxy; import software.amazon.kinesis.leases.ShardInfo; -import software.amazon.kinesis.processor.IRecordProcessor; +import software.amazon.kinesis.processor.RecordProcessor; import software.amazon.kinesis.retrieval.GetRecordsCache; import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; import software.amazon.kinesis.utils.TestStreamlet; @@ -57,7 +57,7 @@ public class ShutdownTaskTest { private final String shardId = "shardId-0000397840"; private boolean cleanupLeasesOfCompletedShards = false; private boolean ignoreUnexpectedChildShards = false; - private IRecordProcessor recordProcessor; + private RecordProcessor recordProcessor; private ShardInfo shardInfo; private ShutdownTask task; diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/retrieval/KinesisDataFetcherTest.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/retrieval/KinesisDataFetcherTest.java index 896a94e3..ab4f530f 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/retrieval/KinesisDataFetcherTest.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/retrieval/KinesisDataFetcherTest.java @@ -56,7 +56,7 @@ import com.amazonaws.services.kinesis.model.ResourceNotFoundException; import com.amazonaws.services.kinesis.model.ShardIteratorType; import software.amazon.kinesis.checkpoint.SentinelCheckpoint; -import software.amazon.kinesis.processor.ICheckpoint; +import software.amazon.kinesis.processor.Checkpointer; import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; /** @@ -134,7 +134,7 @@ public class KinesisDataFetcherTest { @Test public void testadvanceIteratorTo() throws KinesisClientLibException { - final ICheckpoint checkpoint = mock(ICheckpoint.class); + final Checkpointer checkpoint = mock(Checkpointer.class); final String iteratorA = "foo"; final String iteratorB = "bar"; final String seqA = "123"; @@ -398,7 +398,7 @@ public class KinesisDataFetcherTest { when(amazonKinesis.getRecords(recordsCaptor.capture())) .thenReturn(new GetRecordsResult().withRecords(expectedRecords)); - ICheckpoint checkpoint = mock(ICheckpoint.class); + Checkpointer checkpoint = mock(Checkpointer.class); when(checkpoint.getCheckpoint(SHARD_ID)).thenReturn(new ExtendedSequenceNumber(seqNo)); final GetRecordsRetrievalStrategy getRecordsRetrievalStrategy = diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/utils/TestStreamlet.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/utils/TestStreamlet.java index 20dade1f..f19a501a 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/utils/TestStreamlet.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/utils/TestStreamlet.java @@ -29,8 +29,8 @@ import com.amazonaws.services.kinesis.clientlibrary.exceptions.ThrottlingExcepti import software.amazon.kinesis.leases.ShardSequenceVerifier; import software.amazon.kinesis.lifecycle.ShutdownReason; import software.amazon.kinesis.processor.IRecordProcessorCheckpointer; -import software.amazon.kinesis.processor.IRecordProcessor; -import software.amazon.kinesis.processor.IShutdownNotificationAware; +import software.amazon.kinesis.processor.RecordProcessor; +import software.amazon.kinesis.processor.ShutdownNotificationAware; import software.amazon.kinesis.lifecycle.InitializationInput; import software.amazon.kinesis.lifecycle.ProcessRecordsInput; import software.amazon.kinesis.lifecycle.ShutdownInput; @@ -42,7 +42,7 @@ import lombok.extern.slf4j.Slf4j; * Streamlet that tracks records it's seen - useful for testing. */ @Slf4j -public class TestStreamlet implements IRecordProcessor, IShutdownNotificationAware { +public class TestStreamlet implements RecordProcessor, ShutdownNotificationAware { private List records = new ArrayList(); private Set processedSeqNums = new HashSet(); // used for deduping diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/utils/TestStreamletFactory.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/utils/TestStreamletFactory.java index c65fa85b..88df40df 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/utils/TestStreamletFactory.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/utils/TestStreamletFactory.java @@ -19,13 +19,13 @@ import java.util.List; import java.util.concurrent.Semaphore; import software.amazon.kinesis.leases.ShardSequenceVerifier; -import software.amazon.kinesis.processor.IRecordProcessor; -import software.amazon.kinesis.processor.IRecordProcessorFactory; +import software.amazon.kinesis.processor.RecordProcessor; +import software.amazon.kinesis.processor.RecordProcessorFactory; /** * Factory for TestStreamlet record processors. */ -public class TestStreamletFactory implements IRecordProcessorFactory { +public class TestStreamletFactory implements RecordProcessorFactory { // Will be passed to the TestStreamlet. Can be used to check if all records have been processed. private Semaphore semaphore; @@ -41,7 +41,7 @@ public class TestStreamletFactory implements IRecordProcessorFactory { } @Override - public synchronized IRecordProcessor createProcessor() { + public synchronized RecordProcessor createProcessor() { TestStreamlet processor = new TestStreamlet(semaphore, shardSequenceVerifier); testStreamlets.add(processor); return processor;