T appendUserAgent(final T builder) {
- return (T) builder
- .overrideConfiguration(
- AwsRequestOverrideConfiguration.builder()
- .addApiName(ApiName.builder().name(RetrievalConfig.KINESIS_CLIENT_LIB_USER_AGENT)
- .version(RetrievalConfig.KINESIS_CLIENT_LIB_USER_AGENT_VERSION).build())
- .build());
- }
-
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/CoordinatorConfig.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/CoordinatorConfig.java
deleted file mode 100644
index 6098b2fa..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/CoordinatorConfig.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-
-package software.amazon.kinesis.coordinator;
-
-import lombok.Data;
-import lombok.NonNull;
-import lombok.experimental.Accessors;
-import software.amazon.kinesis.leases.NoOpShardPrioritization;
-import software.amazon.kinesis.leases.ShardPrioritization;
-
-/**
- * Used by the KCL to configure the coordinator.
- */
-@Data
-@Accessors(fluent = true)
-public class CoordinatorConfig {
- /**
- * Application name used by checkpointer to checkpoint.
- *
- * @return String
- */
- @NonNull
- private final String applicationName;
-
- /**
- * The maximum number of attempts to initialize the Scheduler
- *
- * Default value: 20
- */
- private int maxInitializationAttempts = 20;
-
- /**
- * Interval in milliseconds between polling to check for parent shard completion.
- * Polling frequently will take up more DynamoDB IOPS (when there are leases for shards waiting on
- * completion of parent shards).
- *
- * Default value: 10000L
- */
- private long parentShardPollIntervalMillis = 10000L;
-
- /**
- * The Scheduler will skip shard sync during initialization if there are one or more leases in the lease table. This
- * assumes that the shards and leases are in-sync. This enables customers to choose faster startup times (e.g.
- * during incremental deployments of an application).
- *
- * Default value: false
- */
- private boolean skipShardSyncAtWorkerInitializationIfLeasesExist = false;
-
- /**
- * The number of milliseconds between polling of the shard consumer for triggering state changes, and health checks.
- *
- * Default value: 1000 milliseconds
- */
- private long shardConsumerDispatchPollIntervalMillis = 1000L;
-
- /**
- * Shard prioritization strategy.
- *
- * Default value: {@link NoOpShardPrioritization}
- */
- private ShardPrioritization shardPrioritization = new NoOpShardPrioritization();
-
- /**
- * WorkerStateChangeListener to be used by the Scheduler.
- *
- * Default value: {@link NoOpWorkerStateChangeListener}
- */
- private WorkerStateChangeListener workerStateChangeListener = new NoOpWorkerStateChangeListener();
-
- /**
- * GracefulShutdownCoordinator to be used by the Scheduler.
- *
- * Default value: {@link GracefulShutdownCoordinator}
- */
- private GracefulShutdownCoordinator gracefulShutdownCoordinator = new GracefulShutdownCoordinator();
-
- private CoordinatorFactory coordinatorFactory = new SchedulerCoordinatorFactory();
-
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/CoordinatorFactory.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/CoordinatorFactory.java
deleted file mode 100644
index 7a055bc7..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/CoordinatorFactory.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-
-package software.amazon.kinesis.coordinator;
-
-import java.util.concurrent.ExecutorService;
-
-import software.amazon.kinesis.checkpoint.ShardRecordProcessorCheckpointer;
-import software.amazon.kinesis.leases.ShardInfo;
-import software.amazon.kinesis.processor.Checkpointer;
-
-/**
- * Used in the process of configuring and providing instances to the {@link Scheduler}
- */
-public interface CoordinatorFactory {
- /**
- * Creates the executor service to be used by the Scheduler.
- *
- * @return ExecutorService
- */
- ExecutorService createExecutorService();
-
- /**
- * Creates GracefulShutdownCoordinator to be used by the Scheduler.
- *
- * Method Deprecated
- *
- * Note: This method has been deprecated, and will be removed in a future release. Use the configuration in
- * {@link CoordinatorConfig#gracefulShutdownCoordinator}. Set the
- * {@link CoordinatorConfig#gracefulShutdownCoordinator} to null in order to use this method.
- *
- * Resolution Order
- *
- * - {@link CoordinatorConfig#gracefulShutdownCoordinator()}
- * - {@link CoordinatorFactory#createGracefulShutdownCoordinator()}
- *
- *
- *
- * @return a {@link GracefulShutdownCoordinator} that manages the process of shutting down the scheduler.
- */
- @Deprecated
- default GracefulShutdownCoordinator createGracefulShutdownCoordinator() {
- return new GracefulShutdownCoordinator();
- }
-
- /**
- * Creates a WorkerStateChangeListener to be used by the Scheduler.
- *
- * Method Deprecated
- *
- * Note: This method has been deprecated, and will be removed in a future release. Use the configuration in
- * {@link CoordinatorConfig#workerStateChangeListener}. Set the
- * {@link CoordinatorConfig#workerStateChangeListener} to null in order to use this method.
- *
- *
- * Resolution Order
- *
- * - {@link CoordinatorConfig#workerStateChangeListener()}
- * - {@link CoordinatorFactory#createWorkerStateChangeListener()}
- *
- *
- * @return a {@link WorkerStateChangeListener} instance that will be notified for specific {@link Scheduler} steps.
- */
- @Deprecated
- default WorkerStateChangeListener createWorkerStateChangeListener() {
- return new NoOpWorkerStateChangeListener();
- }
-
- /**
- * Creates a RecordProcessorChedckpointer to be used by the Scheduler.
- *
- * @param shardInfo ShardInfo to be used in order to create the ShardRecordProcessorCheckpointer
- * @param checkpoint Checkpointer to be used in order to create Shardthe RecordProcessorCheckpointer
- * @return ShardRecordProcessorCheckpointer
- */
- ShardRecordProcessorCheckpointer createRecordProcessorCheckpointer(ShardInfo shardInfo, Checkpointer checkpoint);
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/GracefulShutdownContext.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/GracefulShutdownContext.java
deleted file mode 100644
index 02fca78a..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/GracefulShutdownContext.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-package software.amazon.kinesis.coordinator;
-
-import lombok.Data;
-import lombok.experimental.Accessors;
-
-import java.util.concurrent.CountDownLatch;
-
-@Data
-@Accessors(fluent = true)
-class GracefulShutdownContext {
- private final CountDownLatch shutdownCompleteLatch;
- private final CountDownLatch notificationCompleteLatch;
- private final Scheduler scheduler;
-
- static GracefulShutdownContext SHUTDOWN_ALREADY_COMPLETED = new GracefulShutdownContext(null, null, null);
-
- boolean isShutdownAlreadyCompleted() {
- return shutdownCompleteLatch == null && notificationCompleteLatch == null && scheduler == null;
- }
-
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/GracefulShutdownCoordinator.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/GracefulShutdownCoordinator.java
deleted file mode 100644
index 6dbc534d..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/GracefulShutdownCoordinator.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-package software.amazon.kinesis.coordinator;
-
-import java.util.concurrent.Callable;
-import java.util.concurrent.Future;
-import java.util.concurrent.FutureTask;
-import java.util.concurrent.TimeUnit;
-
-import lombok.extern.slf4j.Slf4j;
-
-class GracefulShutdownCoordinator {
-
- Future startGracefulShutdown(Callable shutdownCallable) {
- FutureTask task = new FutureTask<>(shutdownCallable);
- Thread shutdownThread = new Thread(task, "RequestedShutdownThread");
- shutdownThread.start();
- return task;
-
- }
-
- Callable createGracefulShutdownCallable(Callable startWorkerShutdown) {
- return new GracefulShutdownCallable(startWorkerShutdown);
- }
-
- @Slf4j
- static class GracefulShutdownCallable implements Callable {
- private final Callable startWorkerShutdown;
-
- GracefulShutdownCallable(Callable startWorkerShutdown) {
- this.startWorkerShutdown = startWorkerShutdown;
- }
-
- private boolean isWorkerShutdownComplete(GracefulShutdownContext context) {
- return context.scheduler().shutdownComplete() || context.scheduler().shardInfoShardConsumerMap().isEmpty();
- }
-
- private String awaitingLogMessage(GracefulShutdownContext context) {
- long awaitingNotification = context.notificationCompleteLatch().getCount();
- long awaitingFinalShutdown = context.shutdownCompleteLatch().getCount();
-
- return String.format(
- "Waiting for %d record process to complete shutdown notification, and %d record processor to complete final shutdown ",
- awaitingNotification, awaitingFinalShutdown);
- }
-
- private String awaitingFinalShutdownMessage(GracefulShutdownContext context) {
- long outstanding = context.shutdownCompleteLatch().getCount();
- return String.format("Waiting for %d record processors to complete final shutdown", outstanding);
- }
-
- private boolean waitForRecordProcessors(GracefulShutdownContext context) {
-
- //
- // Awaiting for all ShardConsumer/RecordProcessors to be notified that a shutdown has been requested.
- // There is the possibility of a race condition where a lease is terminated after the shutdown request
- // notification is started, but before the ShardConsumer is sent the notification. In this case the
- // ShardConsumer would start the lease loss shutdown, and may never call the notification methods.
- //
- try {
- while (!context.notificationCompleteLatch().await(1, TimeUnit.SECONDS)) {
- if (Thread.interrupted()) {
- throw new InterruptedException();
- }
- log.info(awaitingLogMessage(context));
- if (workerShutdownWithRemaining(context.shutdownCompleteLatch().getCount(), context)) {
- return false;
- }
- }
- } catch (InterruptedException ie) {
- log.warn("Interrupted while waiting for notification complete, terminating shutdown. {}",
- awaitingLogMessage(context));
- return false;
- }
-
- if (Thread.interrupted()) {
- log.warn("Interrupted before worker shutdown, terminating shutdown");
- return false;
- }
-
- //
- // Once all record processors have been notified of the shutdown it is safe to allow the worker to
- // start its shutdown behavior. Once shutdown starts it will stop renewer, and drop any remaining leases.
- //
- context.scheduler().shutdown();
-
- if (Thread.interrupted()) {
- log.warn("Interrupted after worker shutdown, terminating shutdown");
- return false;
- }
-
- //
- // Want to wait for all the remaining ShardConsumers/ShardRecordProcessor's to complete their final shutdown
- // processing. This should really be a no-op since as part of the notification completion the lease for
- // ShardConsumer is terminated.
- //
- try {
- while (!context.shutdownCompleteLatch().await(1, TimeUnit.SECONDS)) {
- if (Thread.interrupted()) {
- throw new InterruptedException();
- }
- log.info(awaitingFinalShutdownMessage(context));
- if (workerShutdownWithRemaining(context.shutdownCompleteLatch().getCount(), context)) {
- return false;
- }
- }
- } catch (InterruptedException ie) {
- log.warn("Interrupted while waiting for shutdown completion, terminating shutdown. {}",
- awaitingFinalShutdownMessage(context));
- return false;
- }
- return true;
- }
-
- /**
- * This checks to see if the worker has already hit it's shutdown target, while there is outstanding record
- * processors. This maybe a little racy due to when the value of outstanding is retrieved. In general though the
- * latch should be decremented before the shutdown completion.
- *
- * @param outstanding
- * the number of record processor still awaiting shutdown.
- */
- private boolean workerShutdownWithRemaining(long outstanding, GracefulShutdownContext context) {
- if (isWorkerShutdownComplete(context)) {
- if (outstanding != 0) {
- log.info("Shutdown completed, but shutdownCompleteLatch still had outstanding {} with a current"
- + " value of {}. shutdownComplete: {} -- Consumer Map: {}", outstanding,
- context.shutdownCompleteLatch().getCount(), context.scheduler().shutdownComplete(),
- context.scheduler().shardInfoShardConsumerMap().size());
- return true;
- }
- }
- return false;
- }
-
- @Override
- public Boolean call() throws Exception {
- GracefulShutdownContext context;
- try {
- context = startWorkerShutdown.call();
- } catch (Exception ex) {
- log.warn("Caught exception while requesting initial worker shutdown.", ex);
- throw ex;
- }
- return context.isShutdownAlreadyCompleted() || waitForRecordProcessors(context);
- }
- }
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/NoOpWorkerStateChangeListener.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/NoOpWorkerStateChangeListener.java
deleted file mode 100644
index f316b351..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/NoOpWorkerStateChangeListener.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-package software.amazon.kinesis.coordinator;
-
-public class NoOpWorkerStateChangeListener implements WorkerStateChangeListener {
-
- /**
- * Empty constructor for NoOp Worker State Change Listener
- */
- public NoOpWorkerStateChangeListener() {
-
- }
-
- @Override
- public void onWorkerStateChange(WorkerState newState) {
-
- }
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/Scheduler.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/Scheduler.java
deleted file mode 100644
index df7fdda4..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/Scheduler.java
+++ /dev/null
@@ -1,657 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-
-package software.amazon.kinesis.coordinator;
-
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-
-import com.google.common.annotations.VisibleForTesting;
-
-import lombok.AccessLevel;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.NonNull;
-import lombok.experimental.Accessors;
-import lombok.extern.slf4j.Slf4j;
-import software.amazon.kinesis.checkpoint.CheckpointConfig;
-import software.amazon.kinesis.checkpoint.ShardRecordProcessorCheckpointer;
-import software.amazon.kinesis.common.InitialPositionInStreamExtended;
-import software.amazon.kinesis.leases.Lease;
-import software.amazon.kinesis.leases.LeaseCoordinator;
-import software.amazon.kinesis.leases.LeaseManagementConfig;
-import software.amazon.kinesis.leases.LeaseRefresher;
-import software.amazon.kinesis.leases.ShardDetector;
-import software.amazon.kinesis.leases.ShardInfo;
-import software.amazon.kinesis.leases.ShardPrioritization;
-import software.amazon.kinesis.leases.ShardSyncTask;
-import software.amazon.kinesis.leases.ShardSyncTaskManager;
-import software.amazon.kinesis.leases.HierarchicalShardSyncer;
-import software.amazon.kinesis.leases.dynamodb.DynamoDBLeaseCoordinator;
-import software.amazon.kinesis.leases.exceptions.LeasingException;
-import software.amazon.kinesis.lifecycle.LifecycleConfig;
-import software.amazon.kinesis.lifecycle.ShardConsumer;
-import software.amazon.kinesis.lifecycle.ShardConsumerArgument;
-import software.amazon.kinesis.lifecycle.ShardConsumerShutdownNotification;
-import software.amazon.kinesis.lifecycle.ShutdownNotification;
-import software.amazon.kinesis.lifecycle.ShutdownReason;
-import software.amazon.kinesis.lifecycle.TaskResult;
-import software.amazon.kinesis.metrics.CloudWatchMetricsFactory;
-import software.amazon.kinesis.metrics.MetricsCollectingTaskDecorator;
-import software.amazon.kinesis.metrics.MetricsConfig;
-import software.amazon.kinesis.metrics.MetricsFactory;
-import software.amazon.kinesis.processor.Checkpointer;
-import software.amazon.kinesis.processor.ProcessorConfig;
-import software.amazon.kinesis.processor.ShardRecordProcessorFactory;
-import software.amazon.kinesis.processor.ShutdownNotificationAware;
-import software.amazon.kinesis.retrieval.AggregatorUtil;
-import software.amazon.kinesis.retrieval.RecordsPublisher;
-import software.amazon.kinesis.retrieval.RetrievalConfig;
-
-/**
- *
- */
-@Getter
-@Accessors(fluent = true)
-@Slf4j
-public class Scheduler implements Runnable {
-
- private SchedulerLog slog = new SchedulerLog();
-
- private final CheckpointConfig checkpointConfig;
- private final CoordinatorConfig coordinatorConfig;
- private final LeaseManagementConfig leaseManagementConfig;
- private final LifecycleConfig lifecycleConfig;
- private final MetricsConfig metricsConfig;
- private final ProcessorConfig processorConfig;
- private final RetrievalConfig retrievalConfig;
-
- private final String applicationName;
- private final int maxInitializationAttempts;
- private final Checkpointer checkpoint;
- private final long shardConsumerDispatchPollIntervalMillis;
- // Backoff time when polling to check if application has finished processing
- // parent shards
- private final long parentShardPollIntervalMillis;
- private final ExecutorService executorService;
- // private final GetRecordsRetrievalStrategy getRecordsRetrievalStrategy;
- private final LeaseCoordinator leaseCoordinator;
- private final ShardSyncTaskManager shardSyncTaskManager;
- private final ShardPrioritization shardPrioritization;
- private final boolean cleanupLeasesUponShardCompletion;
- private final boolean skipShardSyncAtWorkerInitializationIfLeasesExist;
- private final GracefulShutdownCoordinator gracefulShutdownCoordinator;
- private final WorkerStateChangeListener workerStateChangeListener;
- private final InitialPositionInStreamExtended initialPosition;
- private final MetricsFactory metricsFactory;
- private final long failoverTimeMillis;
- private final long taskBackoffTimeMillis;
- private final String streamName;
- private final long listShardsBackoffTimeMillis;
- private final int maxListShardsRetryAttempts;
- private final LeaseRefresher leaseRefresher;
- private final ShardDetector shardDetector;
- private final boolean ignoreUnexpetedChildShards;
- private final AggregatorUtil aggregatorUtil;
- private final HierarchicalShardSyncer hierarchicalShardSyncer;
-
- // Holds consumers for shards the worker is currently tracking. Key is shard
- // info, value is ShardConsumer.
- private ConcurrentMap shardInfoShardConsumerMap = new ConcurrentHashMap<>();
-
- private volatile boolean shutdown;
- private volatile long shutdownStartTimeMillis;
- private volatile boolean shutdownComplete = false;
-
- private final Object lock = new Object();
-
- /**
- * Used to ensure that only one requestedShutdown is in progress at a time.
- */
- private Future gracefulShutdownFuture;
- @VisibleForTesting
- protected boolean gracefuleShutdownStarted = false;
-
- public Scheduler(@NonNull final CheckpointConfig checkpointConfig,
- @NonNull final CoordinatorConfig coordinatorConfig,
- @NonNull final LeaseManagementConfig leaseManagementConfig,
- @NonNull final LifecycleConfig lifecycleConfig,
- @NonNull final MetricsConfig metricsConfig,
- @NonNull final ProcessorConfig processorConfig,
- @NonNull final RetrievalConfig retrievalConfig) {
- this.checkpointConfig = checkpointConfig;
- this.coordinatorConfig = coordinatorConfig;
- this.leaseManagementConfig = leaseManagementConfig;
- this.lifecycleConfig = lifecycleConfig;
- this.metricsConfig = metricsConfig;
- this.processorConfig = processorConfig;
- this.retrievalConfig = retrievalConfig;
-
- this.applicationName = this.coordinatorConfig.applicationName();
- this.maxInitializationAttempts = this.coordinatorConfig.maxInitializationAttempts();
- this.metricsFactory = this.metricsConfig.metricsFactory();
- this.leaseCoordinator = this.leaseManagementConfig.leaseManagementFactory()
- .createLeaseCoordinator(this.metricsFactory);
- this.leaseRefresher = this.leaseCoordinator.leaseRefresher();
-
- //
- // TODO: Figure out what to do with lease manage <=> checkpoint relationship
- //
- this.checkpoint = this.checkpointConfig.checkpointFactory().createCheckpointer(this.leaseCoordinator,
- this.leaseRefresher);
-
- //
- // TODO: Move this configuration to lifecycle
- //
- this.shardConsumerDispatchPollIntervalMillis = this.coordinatorConfig.shardConsumerDispatchPollIntervalMillis();
- this.parentShardPollIntervalMillis = this.coordinatorConfig.parentShardPollIntervalMillis();
- this.executorService = this.coordinatorConfig.coordinatorFactory().createExecutorService();
-
- this.shardSyncTaskManager = this.leaseManagementConfig.leaseManagementFactory()
- .createShardSyncTaskManager(this.metricsFactory);
- this.shardPrioritization = this.coordinatorConfig.shardPrioritization();
- this.cleanupLeasesUponShardCompletion = this.leaseManagementConfig.cleanupLeasesUponShardCompletion();
- this.skipShardSyncAtWorkerInitializationIfLeasesExist =
- this.coordinatorConfig.skipShardSyncAtWorkerInitializationIfLeasesExist();
- if (coordinatorConfig.gracefulShutdownCoordinator() != null) {
- this.gracefulShutdownCoordinator = coordinatorConfig.gracefulShutdownCoordinator();
- } else {
- this.gracefulShutdownCoordinator = this.coordinatorConfig.coordinatorFactory()
- .createGracefulShutdownCoordinator();
- }
- if (coordinatorConfig.workerStateChangeListener() != null) {
- this.workerStateChangeListener = coordinatorConfig.workerStateChangeListener();
- } else {
- this.workerStateChangeListener = this.coordinatorConfig.coordinatorFactory()
- .createWorkerStateChangeListener();
- }
- this.initialPosition = retrievalConfig.initialPositionInStreamExtended();
- this.failoverTimeMillis = this.leaseManagementConfig.failoverTimeMillis();
- this.taskBackoffTimeMillis = this.lifecycleConfig.taskBackoffTimeMillis();
-// this.retryGetRecordsInSeconds = this.retrievalConfig.retryGetRecordsInSeconds();
-// this.maxGetRecordsThreadPool = this.retrievalConfig.maxGetRecordsThreadPool();
- this.streamName = this.retrievalConfig.streamName();
- this.listShardsBackoffTimeMillis = this.retrievalConfig.listShardsBackoffTimeInMillis();
- this.maxListShardsRetryAttempts = this.retrievalConfig.maxListShardsRetryAttempts();
- this.shardDetector = this.shardSyncTaskManager.shardDetector();
- this.ignoreUnexpetedChildShards = this.leaseManagementConfig.ignoreUnexpectedChildShards();
- this.aggregatorUtil = this.lifecycleConfig.aggregatorUtil();
- this.hierarchicalShardSyncer = leaseManagementConfig.hierarchicalShardSyncer();
- }
-
- /**
- * Start consuming data from the stream, and pass it to the application record processors.
- */
- @Override
- public void run() {
- if (shutdown) {
- return;
- }
-
- try {
- initialize();
- log.info("Initialization complete. Starting worker loop.");
- } catch (RuntimeException e) {
- log.error("Unable to initialize after {} attempts. Shutting down.", maxInitializationAttempts, e);
- workerStateChangeListener.onAllInitializationAttemptsFailed(e);
- shutdown();
- }
-
- while (!shouldShutdown()) {
- runProcessLoop();
- }
-
- finalShutdown();
- log.info("Worker loop is complete. Exiting from worker.");
- }
-
- private void initialize() {
- synchronized (lock) {
- workerStateChangeListener.onWorkerStateChange(WorkerStateChangeListener.WorkerState.INITIALIZING);
- boolean isDone = false;
- Exception lastException = null;
-
- for (int i = 0; (!isDone) && (i < maxInitializationAttempts); i++) {
- try {
- log.info("Initialization attempt {}", (i + 1));
- log.info("Initializing LeaseCoordinator");
- leaseCoordinator.initialize();
-
- TaskResult result = null;
- if (!skipShardSyncAtWorkerInitializationIfLeasesExist || leaseRefresher.isLeaseTableEmpty()) {
- log.info("Syncing Kinesis shard info");
- ShardSyncTask shardSyncTask = new ShardSyncTask(shardDetector, leaseRefresher, initialPosition,
- cleanupLeasesUponShardCompletion, ignoreUnexpetedChildShards, 0L, hierarchicalShardSyncer,
- metricsFactory);
- result = new MetricsCollectingTaskDecorator(shardSyncTask, metricsFactory).call();
- } else {
- log.info("Skipping shard sync per configuration setting (and lease table is not empty)");
- }
-
- if (result == null || result.getException() == null) {
- if (!leaseCoordinator.isRunning()) {
- log.info("Starting LeaseCoordinator");
- leaseCoordinator.start();
- } else {
- log.info("LeaseCoordinator is already running. No need to start it.");
- }
- isDone = true;
- } else {
- lastException = result.getException();
- }
- } catch (LeasingException e) {
- log.error("Caught exception when initializing LeaseCoordinator", e);
- lastException = e;
- } catch (Exception e) {
- lastException = e;
- }
-
- try {
- Thread.sleep(parentShardPollIntervalMillis);
- } catch (InterruptedException e) {
- log.debug("Sleep interrupted while initializing worker.");
- }
- }
-
- if (!isDone) {
- throw new RuntimeException(lastException);
- }
- workerStateChangeListener.onWorkerStateChange(WorkerStateChangeListener.WorkerState.STARTED);
- }
- }
-
- @VisibleForTesting
- void runProcessLoop() {
- try {
- boolean foundCompletedShard = false;
- Set assignedShards = new HashSet<>();
- for (ShardInfo shardInfo : getShardInfoForAssignments()) {
- ShardConsumer shardConsumer = createOrGetShardConsumer(shardInfo,
- processorConfig.shardRecordProcessorFactory());
-
- if (shardConsumer.isShutdown() && shardConsumer.shutdownReason().equals(ShutdownReason.SHARD_END)) {
- foundCompletedShard = true;
- } else {
- shardConsumer.executeLifecycle();
- }
- assignedShards.add(shardInfo);
- }
-
- if (foundCompletedShard) {
- shardSyncTaskManager.syncShardAndLeaseInfo();
- }
-
- // clean up shard consumers for unassigned shards
- cleanupShardConsumers(assignedShards);
-
- slog.info("Sleeping ...");
- Thread.sleep(shardConsumerDispatchPollIntervalMillis);
- } catch (Exception e) {
- log.error("Worker.run caught exception, sleeping for {} milli seconds!",
- String.valueOf(shardConsumerDispatchPollIntervalMillis), e);
- try {
- Thread.sleep(shardConsumerDispatchPollIntervalMillis);
- } catch (InterruptedException ex) {
- log.info("Worker: sleep interrupted after catching exception ", ex);
- }
- }
- slog.resetInfoLogging();
- }
-
- /**
- * Returns whether worker can shutdown immediately. Note that this method is called from Worker's {{@link #run()}
- * method before every loop run, so method must do minimum amount of work to not impact shard processing timings.
- *
- * @return Whether worker should shutdown immediately.
- */
- @VisibleForTesting
- boolean shouldShutdown() {
- if (executorService.isShutdown()) {
- log.error("Worker executor service has been shutdown, so record processors cannot be shutdown.");
- return true;
- }
- if (shutdown) {
- if (shardInfoShardConsumerMap.isEmpty()) {
- log.info("All record processors have been shutdown successfully.");
- return true;
- }
- if ((System.currentTimeMillis() - shutdownStartTimeMillis) >= failoverTimeMillis) {
- log.info("Lease failover time is reached, so forcing shutdown.");
- return true;
- }
- }
- return false;
- }
-
- /**
- * Requests a graceful shutdown of the worker, notifying record processors, that implement
- * {@link ShutdownNotificationAware}, of the impending shutdown. This gives the record processor a final chance to
- * checkpoint.
- *
- * This will only create a single shutdown future. Additional attempts to start a graceful shutdown will return the
- * previous future.
- *
- * It's possible that a record processor won't be notify before being shutdown. This can occur if the lease is
- * lost after requesting shutdown, but before the notification is dispatched.
- *
- * Requested Shutdown Process
When a shutdown process is requested it operates slightly differently to
- * allow the record processors a chance to checkpoint a final time.
- *
- * - Call to request shutdown invoked.
- * - Worker stops attempting to acquire new leases
- * - Record Processor Shutdown Begins
- *
- * - Record processor is notified of the impending shutdown, and given a final chance to checkpoint
- * - The lease for the record processor is then dropped.
- * - The record processor enters into an idle state waiting for the worker to complete final termination
- * - The worker will detect a record processor that has lost it's lease, and will terminate the record processor
- * with {@link ShutdownReason#LEASE_LOST}
- *
- *
- * - The worker will shutdown all record processors.
- * - Once all record processors have been terminated, the worker will terminate all owned resources.
- * - Once the worker shutdown is complete, the returned future is completed.
- *
- *
- * @return a future that will be set once the shutdown has completed. True indicates that the graceful shutdown
- * completed successfully. A false value indicates that a non-exception case caused the shutdown process to
- * terminate early.
- */
- public Future startGracefulShutdown() {
- synchronized (this) {
- if (gracefulShutdownFuture == null) {
- gracefulShutdownFuture = gracefulShutdownCoordinator
- .startGracefulShutdown(createGracefulShutdownCallable());
- }
- }
- return gracefulShutdownFuture;
- }
-
- /**
- * Creates a callable that will execute the graceful shutdown process. This callable can be used to execute graceful
- * shutdowns in your own executor, or execute the shutdown synchronously.
- *
- * @return a callable that run the graceful shutdown process. This may return a callable that return true if the
- * graceful shutdown has already been completed.
- * @throws IllegalStateException
- * thrown by the callable if another callable has already started the shutdown process.
- */
- public Callable createGracefulShutdownCallable() {
- if (shutdownComplete()) {
- return () -> true;
- }
- Callable startShutdown = createWorkerShutdownCallable();
- return gracefulShutdownCoordinator.createGracefulShutdownCallable(startShutdown);
- }
-
- public boolean hasGracefulShutdownStarted() {
- return gracefuleShutdownStarted;
- }
-
- @VisibleForTesting
- Callable createWorkerShutdownCallable() {
- return () -> {
- synchronized (this) {
- if (this.gracefuleShutdownStarted) {
- throw new IllegalStateException("Requested shutdown has already been started");
- }
- this.gracefuleShutdownStarted = true;
- }
- //
- // Stop accepting new leases. Once we do this we can be sure that
- // no more leases will be acquired.
- //
- leaseCoordinator.stopLeaseTaker();
-
- Collection leases = leaseCoordinator.getAssignments();
- if (leases == null || leases.isEmpty()) {
- //
- // If there are no leases notification is already completed, but we still need to shutdown the worker.
- //
- this.shutdown();
- return GracefulShutdownContext.SHUTDOWN_ALREADY_COMPLETED;
- }
- CountDownLatch shutdownCompleteLatch = new CountDownLatch(leases.size());
- CountDownLatch notificationCompleteLatch = new CountDownLatch(leases.size());
- for (Lease lease : leases) {
- ShutdownNotification shutdownNotification = new ShardConsumerShutdownNotification(leaseCoordinator,
- lease, notificationCompleteLatch, shutdownCompleteLatch);
- ShardInfo shardInfo = DynamoDBLeaseCoordinator.convertLeaseToAssignment(lease);
- ShardConsumer consumer = shardInfoShardConsumerMap.get(shardInfo);
- if (consumer != null) {
- consumer.gracefulShutdown(shutdownNotification);
- } else {
- //
- // There is a race condition between retrieving the current assignments, and creating the
- // notification. If the a lease is lost in between these two points, we explicitly decrement the
- // notification latches to clear the shutdown.
- //
- notificationCompleteLatch.countDown();
- shutdownCompleteLatch.countDown();
- }
- }
- return new GracefulShutdownContext(shutdownCompleteLatch, notificationCompleteLatch, this);
- };
- }
-
- /**
- * Signals worker to shutdown. Worker will try initiating shutdown of all record processors. Note that if executor
- * services were passed to the worker by the user, worker will not attempt to shutdown those resources.
- *
- * Shutdown Process
When called this will start shutdown of the record processor, and eventually shutdown
- * the worker itself.
- *
- * - Call to start shutdown invoked
- * - Lease coordinator told to stop taking leases, and to drop existing leases.
- * - Worker discovers record processors that no longer have leases.
- * - Worker triggers shutdown with state {@link ShutdownReason#LEASE_LOST}.
- * - Once all record processors are shutdown, worker terminates owned resources.
- * - Shutdown complete.
- *
- */
- public void shutdown() {
- synchronized (lock) {
- if (shutdown) {
- log.warn("Shutdown requested a second time.");
- return;
- }
- log.info("Worker shutdown requested.");
-
- // Set shutdown flag, so Worker.run can start shutdown process.
- shutdown = true;
- shutdownStartTimeMillis = System.currentTimeMillis();
-
- // Stop lease coordinator, so leases are not renewed or stolen from other workers.
- // Lost leases will force Worker to begin shutdown process for all shard consumers in
- // Worker.run().
- leaseCoordinator.stop();
- workerStateChangeListener.onWorkerStateChange(WorkerStateChangeListener.WorkerState.SHUT_DOWN);
- }
- }
-
- /**
- * Perform final shutdown related tasks for the worker including shutting down worker owned executor services,
- * threads, etc.
- */
- private void finalShutdown() {
- log.info("Starting worker's final shutdown.");
-
- if (executorService instanceof SchedulerCoordinatorFactory.SchedulerThreadPoolExecutor) {
- // This should interrupt all active record processor tasks.
- executorService.shutdownNow();
- }
- if (metricsFactory instanceof CloudWatchMetricsFactory) {
- ((CloudWatchMetricsFactory) metricsFactory).shutdown();
- }
- shutdownComplete = true;
- }
-
- private List getShardInfoForAssignments() {
- List assignedStreamShards = leaseCoordinator.getCurrentAssignments();
- List prioritizedShards = shardPrioritization.prioritize(assignedStreamShards);
-
- if ((prioritizedShards != null) && (!prioritizedShards.isEmpty())) {
- if (slog.isInfoEnabled()) {
- StringBuilder builder = new StringBuilder();
- boolean firstItem = true;
- for (ShardInfo shardInfo : prioritizedShards) {
- if (!firstItem) {
- builder.append(", ");
- }
- builder.append(shardInfo.shardId());
- firstItem = false;
- }
- slog.info("Current stream shard assignments: " + builder.toString());
- }
- } else {
- slog.info("No activities assigned");
- }
-
- return prioritizedShards;
- }
-
- /**
- * NOTE: This method is internal/private to the Worker class. It has package access solely for testing.
- *
- * @param shardInfo
- * Kinesis shard info
- * @return ShardConsumer for the shard
- */
- ShardConsumer createOrGetShardConsumer(@NonNull final ShardInfo shardInfo,
- @NonNull final ShardRecordProcessorFactory shardRecordProcessorFactory) {
- ShardConsumer consumer = shardInfoShardConsumerMap.get(shardInfo);
- // Instantiate a new consumer if we don't have one, or the one we
- // had was from an earlier
- // lease instance (and was shutdown). Don't need to create another
- // one if the shard has been
- // completely processed (shutdown reason terminate).
- if ((consumer == null)
- || (consumer.isShutdown() && consumer.shutdownReason().equals(ShutdownReason.LEASE_LOST))) {
- consumer = buildConsumer(shardInfo, shardRecordProcessorFactory);
- shardInfoShardConsumerMap.put(shardInfo, consumer);
- slog.infoForce("Created new shardConsumer for : " + shardInfo);
- }
- return consumer;
- }
-
- protected ShardConsumer buildConsumer(@NonNull final ShardInfo shardInfo,
- @NonNull final ShardRecordProcessorFactory shardRecordProcessorFactory) {
- RecordsPublisher cache = retrievalConfig.retrievalFactory().createGetRecordsCache(shardInfo, metricsFactory);
- ShardRecordProcessorCheckpointer checkpointer = coordinatorConfig.coordinatorFactory().createRecordProcessorCheckpointer(shardInfo,
- checkpoint);
- ShardConsumerArgument argument = new ShardConsumerArgument(shardInfo,
- streamName,
- leaseRefresher,
- executorService,
- cache,
- shardRecordProcessorFactory.shardRecordProcessor(),
- checkpoint,
- checkpointer,
- parentShardPollIntervalMillis,
- taskBackoffTimeMillis,
- skipShardSyncAtWorkerInitializationIfLeasesExist,
- listShardsBackoffTimeMillis,
- maxListShardsRetryAttempts,
- processorConfig.callProcessRecordsEvenForEmptyRecordList(),
- shardConsumerDispatchPollIntervalMillis,
- initialPosition,
- cleanupLeasesUponShardCompletion,
- ignoreUnexpetedChildShards,
- shardDetector,
- aggregatorUtil,
- hierarchicalShardSyncer,
- metricsFactory);
- return new ShardConsumer(cache, executorService, shardInfo, lifecycleConfig.logWarningForTaskAfterMillis(),
- argument, lifecycleConfig.taskExecutionListener());
- }
-
- /**
- * NOTE: This method is internal/private to the Worker class. It has package access solely for testing.
- *
- * This method relies on ShardInfo.equals() method returning true for ShardInfo objects which may have been
- * instantiated with parentShardIds in a different order (and rest of the fields being the equal). For example
- * shardInfo1.equals(shardInfo2) should return true with shardInfo1 and shardInfo2 defined as follows. ShardInfo
- * shardInfo1 = new ShardInfo(shardId1, concurrencyToken1, Arrays.asList("parent1", "parent2")); ShardInfo
- * shardInfo2 = new ShardInfo(shardId1, concurrencyToken1, Arrays.asList("parent2", "parent1"));
- */
- void cleanupShardConsumers(Set assignedShards) {
- for (ShardInfo shard : shardInfoShardConsumerMap.keySet()) {
- if (!assignedShards.contains(shard)) {
- // Shutdown the consumer since we are no longer responsible for
- // the shard.
- ShardConsumer consumer = shardInfoShardConsumerMap.get(shard);
- if (consumer.leaseLost()) {
- shardInfoShardConsumerMap.remove(shard);
- log.debug("Removed consumer for {} as lease has been lost", shard.shardId());
- } else {
- consumer.executeLifecycle();
- }
- }
- }
- }
-
- /**
- * Logger for suppressing too much INFO logging. To avoid too much logging information Worker will output logging at
- * INFO level for a single pass through the main loop every minute. At DEBUG level it will output all INFO logs on
- * every pass.
- */
- @NoArgsConstructor(access = AccessLevel.PRIVATE)
- private static class SchedulerLog {
-
- private long reportIntervalMillis = TimeUnit.MINUTES.toMillis(1);
- private long nextReportTime = System.currentTimeMillis() + reportIntervalMillis;
- private boolean infoReporting;
-
- void info(Object message) {
- if (this.isInfoEnabled()) {
- log.info("{}", message);
- }
- }
-
- void infoForce(Object message) {
- log.info("{}", message);
- }
-
- private boolean isInfoEnabled() {
- return infoReporting;
- }
-
- private void resetInfoLogging() {
- if (infoReporting) {
- // We just logged at INFO level for a pass through worker loop
- if (log.isInfoEnabled()) {
- infoReporting = false;
- nextReportTime = System.currentTimeMillis() + reportIntervalMillis;
- } // else is DEBUG or TRACE so leave reporting true
- } else if (nextReportTime <= System.currentTimeMillis()) {
- infoReporting = true;
- }
- }
- }
-
- @Deprecated
- public Future requestShutdown() {
- return null;
- }
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/SchedulerCoordinatorFactory.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/SchedulerCoordinatorFactory.java
deleted file mode 100644
index 72f830fb..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/SchedulerCoordinatorFactory.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-
-package software.amazon.kinesis.coordinator;
-
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-
-import lombok.Data;
-import lombok.NonNull;
-import software.amazon.kinesis.annotations.KinesisClientInternalApi;
-import software.amazon.kinesis.checkpoint.ShardRecordProcessorCheckpointer;
-import software.amazon.kinesis.leases.ShardInfo;
-import software.amazon.kinesis.processor.Checkpointer;
-
-/**
- *
- */
-@Data
-@KinesisClientInternalApi
-public class SchedulerCoordinatorFactory implements CoordinatorFactory {
- /**
- * {@inheritDoc}
- */
- @Override
- public ExecutorService createExecutorService() {
- return new SchedulerThreadPoolExecutor(
- new ThreadFactoryBuilder().setNameFormat("ShardRecordProcessor-%04d").build());
- }
-
- static class SchedulerThreadPoolExecutor extends ThreadPoolExecutor {
- private static final long DEFAULT_KEEP_ALIVE = 60L;
- SchedulerThreadPoolExecutor(ThreadFactory threadFactory) {
- super(0, Integer.MAX_VALUE, DEFAULT_KEEP_ALIVE, TimeUnit.SECONDS, new SynchronousQueue<>(),
- threadFactory);
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public ShardRecordProcessorCheckpointer createRecordProcessorCheckpointer(@NonNull final ShardInfo shardInfo,
- @NonNull final Checkpointer checkpoint) {
- return new ShardRecordProcessorCheckpointer(shardInfo, checkpoint);
- }
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/WorkerStateChangeListener.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/WorkerStateChangeListener.java
deleted file mode 100644
index 2ca08aa4..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/WorkerStateChangeListener.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-package software.amazon.kinesis.coordinator;
-
-/**
- * A listener for callbacks on changes worker state
- */
-@FunctionalInterface
-public interface WorkerStateChangeListener {
- enum WorkerState {
- CREATED,
- INITIALIZING,
- STARTED,
- SHUT_DOWN
- }
-
- void onWorkerStateChange(WorkerState newState);
-
- default void onAllInitializationAttemptsFailed(Throwable e) {
- }
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/exceptions/ShutdownException.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/exceptions/ShutdownException.java
deleted file mode 100644
index 5a57b11b..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/exceptions/ShutdownException.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-package software.amazon.kinesis.exceptions;
-
-/**
- * The ShardRecordProcessor instance has been shutdown (e.g. and attempts a checkpoint).
- */
-public class ShutdownException extends KinesisClientLibNonRetryableException {
-
- private static final long serialVersionUID = 1L;
-
- /**
- * @param message provides more details about the cause and potential ways to debug/address.
- */
- public ShutdownException(String message) {
- super(message);
- }
-
- /**
- * @param message provides more details about the cause and potential ways to debug/address.
- * @param e Cause of the exception
- */
- public ShutdownException(String message, Exception e) {
- super(message, e);
- }
-
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/exceptions/internal/KinesisClientLibIOException.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/exceptions/internal/KinesisClientLibIOException.java
deleted file mode 100644
index f15a8088..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/exceptions/internal/KinesisClientLibIOException.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-package software.amazon.kinesis.exceptions.internal;
-
-import software.amazon.kinesis.exceptions.KinesisClientLibRetryableException;
-
-/**
- * Thrown when we encounter issues when reading/writing information (e.g. shard information from Kinesis may not be
- * current/complete).
- */
-public class KinesisClientLibIOException extends KinesisClientLibRetryableException {
- private static final long serialVersionUID = 1L;
-
- /**
- * Constructor.
- *
- * @param message Error message.
- */
- public KinesisClientLibIOException(String message) {
- super(message);
- }
-
- /**
- * Constructor.
- *
- * @param message Error message.
- * @param e Cause.
- */
- public KinesisClientLibIOException(String message, Exception e) {
- super(message, e);
- }
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/HierarchicalShardSyncer.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/HierarchicalShardSyncer.java
deleted file mode 100644
index c61bf935..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/HierarchicalShardSyncer.java
+++ /dev/null
@@ -1,756 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-package software.amazon.kinesis.leases;
-
-import java.io.Serializable;
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-
-import org.apache.commons.lang3.StringUtils;
-
-import lombok.NonNull;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import software.amazon.awssdk.services.kinesis.model.Shard;
-import software.amazon.awssdk.utils.CollectionUtils;
-import software.amazon.kinesis.annotations.KinesisClientInternalApi;
-import software.amazon.kinesis.common.InitialPositionInStream;
-import software.amazon.kinesis.common.InitialPositionInStreamExtended;
-import software.amazon.kinesis.exceptions.internal.KinesisClientLibIOException;
-import software.amazon.kinesis.leases.exceptions.DependencyException;
-import software.amazon.kinesis.leases.exceptions.InvalidStateException;
-import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException;
-import software.amazon.kinesis.metrics.MetricsLevel;
-import software.amazon.kinesis.metrics.MetricsScope;
-import software.amazon.kinesis.metrics.MetricsUtil;
-import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber;
-
-/**
- * Helper class to sync leases with shards of the Kinesis stream.
- * It will create new leases/activities when it discovers new Kinesis shards (bootstrap/resharding).
- * It deletes leases for shards that have been trimmed from Kinesis, or if we've completed processing it
- * and begun processing it's child shards.
- */
-@Slf4j
-@KinesisClientInternalApi
-public class HierarchicalShardSyncer {
-
- /**
- * Check and create leases for any new shards (e.g. following a reshard operation). Sync leases with Kinesis shards
- * (e.g. at startup, or when we reach end of a shard).
- *
- * @param shardDetector
- * @param leaseRefresher
- * @param initialPosition
- * @param cleanupLeasesOfCompletedShards
- * @param ignoreUnexpectedChildShards
- * @param scope
- * @throws DependencyException
- * @throws InvalidStateException
- * @throws ProvisionedThroughputException
- * @throws KinesisClientLibIOException
- */
- // CHECKSTYLE:OFF CyclomaticComplexity
- public synchronized void checkAndCreateLeaseForNewShards(@NonNull final ShardDetector shardDetector,
- final LeaseRefresher leaseRefresher, final InitialPositionInStreamExtended initialPosition,
- final boolean cleanupLeasesOfCompletedShards, final boolean ignoreUnexpectedChildShards,
- final MetricsScope scope) throws DependencyException, InvalidStateException,
- ProvisionedThroughputException, KinesisClientLibIOException {
- final List shards = getShardList(shardDetector);
- log.debug("Num shards: {}", shards.size());
-
- final Map shardIdToShardMap = constructShardIdToShardMap(shards);
- final Map> shardIdToChildShardIdsMap = constructShardIdToChildShardIdsMap(
- shardIdToShardMap);
- final Set inconsistentShardIds = findInconsistentShardIds(shardIdToChildShardIdsMap, shardIdToShardMap);
- if (!ignoreUnexpectedChildShards) {
- assertAllParentShardsAreClosed(inconsistentShardIds);
- }
-
- final List currentLeases = leaseRefresher.listLeases();
-
- final List newLeasesToCreate = determineNewLeasesToCreate(shards, currentLeases, initialPosition,
- inconsistentShardIds);
- log.debug("Num new leases to create: {}", newLeasesToCreate.size());
- for (Lease lease : newLeasesToCreate) {
- long startTime = System.currentTimeMillis();
- boolean success = false;
- try {
- leaseRefresher.createLeaseIfNotExists(lease);
- success = true;
- } finally {
- MetricsUtil.addSuccessAndLatency(scope, "CreateLease", success, startTime, MetricsLevel.DETAILED);
- }
- }
-
- final List trackedLeases = new ArrayList<>(currentLeases);
- trackedLeases.addAll(newLeasesToCreate);
- cleanupGarbageLeases(shardDetector, shards, trackedLeases, leaseRefresher);
- if (cleanupLeasesOfCompletedShards) {
- cleanupLeasesOfFinishedShards(currentLeases, shardIdToShardMap, shardIdToChildShardIdsMap, trackedLeases,
- leaseRefresher);
- }
- }
- // CHECKSTYLE:ON CyclomaticComplexity
-
- /** Helper method to detect a race condition between fetching the shards via paginated DescribeStream calls
- * and a reshard operation.
- * @param inconsistentShardIds
- * @throws KinesisClientLibIOException
- */
- private static void assertAllParentShardsAreClosed(final Set inconsistentShardIds)
- throws KinesisClientLibIOException {
- if (!CollectionUtils.isNullOrEmpty(inconsistentShardIds)) {
- final String ids = StringUtils.join(inconsistentShardIds, ' ');
- throw new KinesisClientLibIOException(String.format(
- "%d open child shards (%s) are inconsistent. This can happen due to a race condition between describeStream and a reshard operation.",
- inconsistentShardIds.size(), ids));
- }
- }
-
- /**
- * Helper method to construct the list of inconsistent shards, which are open shards with non-closed ancestor
- * parent(s).
- * @param shardIdToChildShardIdsMap
- * @param shardIdToShardMap
- * @return Set of inconsistent open shard ids for shards having open parents.
- */
- private static Set findInconsistentShardIds(final Map> shardIdToChildShardIdsMap,
- final Map shardIdToShardMap) {
- return shardIdToChildShardIdsMap.entrySet().stream()
- .filter(entry -> entry.getKey() == null
- || shardIdToShardMap.get(entry.getKey()).sequenceNumberRange().endingSequenceNumber() == null)
- .flatMap(entry -> shardIdToChildShardIdsMap.get(entry.getKey()).stream()).collect(Collectors.toSet());
- }
-
- /**
- * Note: this has package level access for testing purposes.
- * Useful for asserting that we don't have an incomplete shard list following a reshard operation.
- * We verify that if the shard is present in the shard list, it is closed and its hash key range
- * is covered by its child shards.
- * @param shardIdsOfClosedShards Id of the shard which is expected to be closed
- * @return ShardIds of child shards (children of the expectedClosedShard)
- * @throws KinesisClientLibIOException
- */
- synchronized void assertClosedShardsAreCoveredOrAbsent(final Map shardIdToShardMap,
- final Map> shardIdToChildShardIdsMap, final Set shardIdsOfClosedShards)
- throws KinesisClientLibIOException {
- final String exceptionMessageSuffix = "This can happen if we constructed the list of shards "
- + " while a reshard operation was in progress.";
-
- for (String shardId : shardIdsOfClosedShards) {
- final Shard shard = shardIdToShardMap.get(shardId);
- if (shard == null) {
- log.info("Shard {} is not present in Kinesis anymore.", shardId);
- continue;
- }
-
- final String endingSequenceNumber = shard.sequenceNumberRange().endingSequenceNumber();
- if (endingSequenceNumber == null) {
- throw new KinesisClientLibIOException("Shard " + shardIdsOfClosedShards
- + " is not closed. " + exceptionMessageSuffix);
- }
-
- final Set childShardIds = shardIdToChildShardIdsMap.get(shardId);
- if (childShardIds == null) {
- throw new KinesisClientLibIOException("Incomplete shard list: Closed shard " + shardId
- + " has no children." + exceptionMessageSuffix);
- }
-
- assertHashRangeOfClosedShardIsCovered(shard, shardIdToShardMap, childShardIds);
- }
- }
-
- private synchronized void assertHashRangeOfClosedShardIsCovered(final Shard closedShard,
- final Map shardIdToShardMap, final Set childShardIds)
- throws KinesisClientLibIOException {
- BigInteger minStartingHashKeyOfChildren = null;
- BigInteger maxEndingHashKeyOfChildren = null;
-
- final BigInteger startingHashKeyOfClosedShard = new BigInteger(closedShard.hashKeyRange().startingHashKey());
- final BigInteger endingHashKeyOfClosedShard = new BigInteger(closedShard.hashKeyRange().endingHashKey());
-
- for (String childShardId : childShardIds) {
- final Shard childShard = shardIdToShardMap.get(childShardId);
- final BigInteger startingHashKey = new BigInteger(childShard.hashKeyRange().startingHashKey());
- if (minStartingHashKeyOfChildren == null || startingHashKey.compareTo(minStartingHashKeyOfChildren) < 0) {
- minStartingHashKeyOfChildren = startingHashKey;
- }
-
- final BigInteger endingHashKey = new BigInteger(childShard.hashKeyRange().endingHashKey());
- if (maxEndingHashKeyOfChildren == null || endingHashKey.compareTo(maxEndingHashKeyOfChildren) > 0) {
- maxEndingHashKeyOfChildren = endingHashKey;
- }
- }
-
- if (minStartingHashKeyOfChildren == null || maxEndingHashKeyOfChildren == null
- || minStartingHashKeyOfChildren.compareTo(startingHashKeyOfClosedShard) > 0
- || maxEndingHashKeyOfChildren.compareTo(endingHashKeyOfClosedShard) < 0) {
- throw new KinesisClientLibIOException(String.format(
- "Incomplete shard list: hash key range of shard %s is not covered by its child shards.",
- closedShard.shardId()));
- }
-
- }
-
- /**
- * Helper method to construct shardId->setOfChildShardIds map.
- * Note: This has package access for testing purposes only.
- * @param shardIdToShardMap
- * @return
- */
- static Map> constructShardIdToChildShardIdsMap(final Map shardIdToShardMap) {
- final Map> shardIdToChildShardIdsMap = new HashMap<>();
-
- for (final Map.Entry entry : shardIdToShardMap.entrySet()) {
- final String shardId = entry.getKey();
- final Shard shard = entry.getValue();
- final String parentShardId = shard.parentShardId();
- if (parentShardId != null && shardIdToShardMap.containsKey(parentShardId)) {
- final Set childShardIds = shardIdToChildShardIdsMap.computeIfAbsent(parentShardId,
- key -> new HashSet<>());
- childShardIds.add(shardId);
- }
-
- final String adjacentParentShardId = shard.adjacentParentShardId();
- if (adjacentParentShardId != null && shardIdToShardMap.containsKey(adjacentParentShardId)) {
- final Set childShardIds = shardIdToChildShardIdsMap.computeIfAbsent(adjacentParentShardId,
- key -> new HashSet<>());
- childShardIds.add(shardId);
- }
- }
- return shardIdToChildShardIdsMap;
- }
-
- private static List getShardList(@NonNull final ShardDetector shardDetector) throws KinesisClientLibIOException {
- final List shards = shardDetector.listShards();
- if (shards == null) {
- throw new KinesisClientLibIOException(
- "Stream is not in ACTIVE OR UPDATING state - will retry getting the shard list.");
- }
- return shards;
- }
-
- /**
- * Determine new leases to create and their initial checkpoint.
- * Note: Package level access only for testing purposes.
- *
- * For each open (no ending sequence number) shard without open parents that doesn't already have a lease,
- * determine if it is a descendent of any shard which is or will be processed (e.g. for which a lease exists):
- * If so, set checkpoint of the shard to TrimHorizon and also create leases for ancestors if needed.
- * If not, set checkpoint of the shard to the initial position specified by the client.
- * To check if we need to create leases for ancestors, we use the following rules:
- * * If we began (or will begin) processing data for a shard, then we must reach end of that shard before
- * we begin processing data from any of its descendants.
- * * A shard does not start processing data until data from all its parents has been processed.
- * Note, if the initial position is LATEST and a shard has two parents and only one is a descendant - we'll create
- * leases corresponding to both the parents - the parent shard which is not a descendant will have
- * its checkpoint set to Latest.
- *
- * We assume that if there is an existing lease for a shard, then either:
- * * we have previously created a lease for its parent (if it was needed), or
- * * the parent shard has expired.
- *
- * For example:
- * Shard structure (each level depicts a stream segment):
- * 0 1 2 3 4 5 - shards till epoch 102
- * \ / \ / | |
- * 6 7 4 5 - shards from epoch 103 - 205
- * \ / | / \
- * 8 4 9 10 - shards from epoch 206 (open - no ending sequenceNumber)
- * Current leases: (3, 4, 5)
- * New leases to create: (2, 6, 7, 8, 9, 10)
- *
- * The leases returned are sorted by the starting sequence number - following the same order
- * when persisting the leases in DynamoDB will ensure that we recover gracefully if we fail
- * before creating all the leases.
- *
- * If a shard has no existing lease, is open, and is a descendant of a parent which is still open, we ignore it
- * here; this happens when the list of shards is inconsistent, which could be due to pagination delay for very
- * high shard count streams (i.e., dynamodb streams for tables with thousands of partitions). This can only
- * currently happen here if ignoreUnexpectedChildShards was true in syncShardleases.
- *
- *
- * @param shards List of all shards in Kinesis (we'll create new leases based on this set)
- * @param currentLeases List of current leases
- * @param initialPosition One of LATEST, TRIM_HORIZON, or AT_TIMESTAMP. We'll start fetching records from that
- * location in the shard (when an application starts up for the first time - and there are no checkpoints).
- * @param inconsistentShardIds Set of child shard ids having open parents.
- * @return List of new leases to create sorted by starting sequenceNumber of the corresponding shard
- */
- static List determineNewLeasesToCreate(final List shards, final List currentLeases,
- final InitialPositionInStreamExtended initialPosition, final Set inconsistentShardIds) {
- final Map shardIdToNewLeaseMap = new HashMap<>();
- final Map shardIdToShardMapOfAllKinesisShards = constructShardIdToShardMap(shards);
-
- final Set shardIdsOfCurrentLeases = currentLeases.stream()
- .peek(lease -> log.debug("Existing lease: {}", lease)).map(Lease::leaseKey).collect(Collectors.toSet());
-
- final List openShards = getOpenShards(shards);
- final Map memoizationContext = new HashMap<>();
-
- // Iterate over the open shards and find those that don't have any lease entries.
- for (Shard shard : openShards) {
- final String shardId = shard.shardId();
- log.debug("Evaluating leases for open shard {} and its ancestors.", shardId);
- if (shardIdsOfCurrentLeases.contains(shardId)) {
- log.debug("Lease for shardId {} already exists. Not creating a lease", shardId);
- } else if (inconsistentShardIds.contains(shardId)) {
- log.info("shardId {} is an inconsistent child. Not creating a lease", shardId);
- } else {
- log.debug("Need to create a lease for shardId {}", shardId);
- final Lease newLease = newKCLLease(shard);
- final boolean isDescendant = checkIfDescendantAndAddNewLeasesForAncestors(shardId, initialPosition,
- shardIdsOfCurrentLeases, shardIdToShardMapOfAllKinesisShards, shardIdToNewLeaseMap,
- memoizationContext);
-
- /**
- * If the shard is a descendant and the specified initial position is AT_TIMESTAMP, then the
- * checkpoint should be set to AT_TIMESTAMP, else to TRIM_HORIZON. For AT_TIMESTAMP, we will add a
- * lease just like we do for TRIM_HORIZON. However we will only return back records with server-side
- * timestamp at or after the specified initial position timestamp.
- *
- * Shard structure (each level depicts a stream segment):
- * 0 1 2 3 4 5 - shards till epoch 102
- * \ / \ / | |
- * 6 7 4 5 - shards from epoch 103 - 205
- * \ / | /\
- * 8 4 9 10 - shards from epoch 206 (open - no ending sequenceNumber)
- *
- * Current leases: empty set
- *
- * For the above example, suppose the initial position in stream is set to AT_TIMESTAMP with
- * timestamp value 206. We will then create new leases for all the shards (with checkpoint set to
- * AT_TIMESTAMP), including the ancestor shards with epoch less than 206. However as we begin
- * processing the ancestor shards, their checkpoints would be updated to SHARD_END and their leases
- * would then be deleted since they won't have records with server-side timestamp at/after 206. And
- * after that we will begin processing the descendant shards with epoch at/after 206 and we will
- * return the records that meet the timestamp requirement for these shards.
- */
- if (isDescendant
- && !initialPosition.getInitialPositionInStream().equals(InitialPositionInStream.AT_TIMESTAMP)) {
- newLease.checkpoint(ExtendedSequenceNumber.TRIM_HORIZON);
- } else {
- newLease.checkpoint(convertToCheckpoint(initialPosition));
- }
- log.debug("Set checkpoint of {} to {}", newLease.leaseKey(), newLease.checkpoint());
- shardIdToNewLeaseMap.put(shardId, newLease);
- }
- }
-
- final List newLeasesToCreate = new ArrayList<>(shardIdToNewLeaseMap.values());
- final Comparator startingSequenceNumberComparator = new StartingSequenceNumberAndShardIdBasedComparator(
- shardIdToShardMapOfAllKinesisShards);
- newLeasesToCreate.sort(startingSequenceNumberComparator);
- return newLeasesToCreate;
- }
-
- /**
- * Determine new leases to create and their initial checkpoint.
- * Note: Package level access only for testing purposes.
- */
- static List determineNewLeasesToCreate(final List shards, final List currentLeases,
- final InitialPositionInStreamExtended initialPosition) {
- final Set inconsistentShardIds = new HashSet<>();
- return determineNewLeasesToCreate(shards, currentLeases, initialPosition, inconsistentShardIds);
- }
-
- /**
- * Note: Package level access for testing purposes only.
- * Check if this shard is a descendant of a shard that is (or will be) processed.
- * Create leases for the ancestors of this shard as required.
- * See javadoc of determineNewLeasesToCreate() for rules and example.
- *
- * @param shardId The shardId to check.
- * @param initialPosition One of LATEST, TRIM_HORIZON, or AT_TIMESTAMP. We'll start fetching records from that
- * location in the shard (when an application starts up for the first time - and there are no checkpoints).
- * @param shardIdsOfCurrentLeases The shardIds for the current leases.
- * @param shardIdToShardMapOfAllKinesisShards ShardId->Shard map containing all shards obtained via DescribeStream.
- * @param shardIdToLeaseMapOfNewShards Add lease POJOs corresponding to ancestors to this map.
- * @param memoizationContext Memoization of shards that have been evaluated as part of the evaluation
- * @return true if the shard is a descendant of any current shard (lease already exists)
- */
- // CHECKSTYLE:OFF CyclomaticComplexity
- static boolean checkIfDescendantAndAddNewLeasesForAncestors(final String shardId,
- final InitialPositionInStreamExtended initialPosition, final Set shardIdsOfCurrentLeases,
- final Map shardIdToShardMapOfAllKinesisShards,
- final Map shardIdToLeaseMapOfNewShards, final Map memoizationContext) {
-
- final Boolean previousValue = memoizationContext.get(shardId);
- if (previousValue != null) {
- return previousValue;
- }
-
- boolean isDescendant = false;
- final Set descendantParentShardIds = new HashSet<>();
-
- if (shardId != null && shardIdToShardMapOfAllKinesisShards.containsKey(shardId)) {
- if (shardIdsOfCurrentLeases.contains(shardId)) {
- // This shard is a descendant of a current shard.
- isDescendant = true;
- // We don't need to add leases of its ancestors,
- // because we'd have done it when creating a lease for this shard.
- } else {
- final Shard shard = shardIdToShardMapOfAllKinesisShards.get(shardId);
- final Set parentShardIds = getParentShardIds(shard, shardIdToShardMapOfAllKinesisShards);
- for (String parentShardId : parentShardIds) {
- // Check if the parent is a descendant, and include its ancestors.
- if (checkIfDescendantAndAddNewLeasesForAncestors(parentShardId, initialPosition,
- shardIdsOfCurrentLeases, shardIdToShardMapOfAllKinesisShards, shardIdToLeaseMapOfNewShards,
- memoizationContext)) {
- isDescendant = true;
- descendantParentShardIds.add(parentShardId);
- log.debug("Parent shard {} is a descendant.", parentShardId);
- } else {
- log.debug("Parent shard {} is NOT a descendant.", parentShardId);
- }
- }
-
- // If this is a descendant, create leases for its parent shards (if they don't exist)
- if (isDescendant) {
- for (String parentShardId : parentShardIds) {
- if (!shardIdsOfCurrentLeases.contains(parentShardId)) {
- log.debug("Need to create a lease for shardId {}", parentShardId);
- Lease lease = shardIdToLeaseMapOfNewShards.get(parentShardId);
- if (lease == null) {
- lease = newKCLLease(shardIdToShardMapOfAllKinesisShards.get(parentShardId));
- shardIdToLeaseMapOfNewShards.put(parentShardId, lease);
- }
-
- if (descendantParentShardIds.contains(parentShardId)
- && !initialPosition.getInitialPositionInStream()
- .equals(InitialPositionInStream.AT_TIMESTAMP)) {
- lease.checkpoint(ExtendedSequenceNumber.TRIM_HORIZON);
- } else {
- lease.checkpoint(convertToCheckpoint(initialPosition));
- }
- }
- }
- } else {
- // This shard should be included, if the customer wants to process all records in the stream or
- // if the initial position is AT_TIMESTAMP. For AT_TIMESTAMP, we will add a lease just like we do
- // for TRIM_HORIZON. However we will only return back records with server-side timestamp at or
- // after the specified initial position timestamp.
- if (initialPosition.getInitialPositionInStream().equals(InitialPositionInStream.TRIM_HORIZON)
- || initialPosition.getInitialPositionInStream()
- .equals(InitialPositionInStream.AT_TIMESTAMP)) {
- isDescendant = true;
- }
- }
-
- }
- }
-
- memoizationContext.put(shardId, isDescendant);
- return isDescendant;
- }
- // CHECKSTYLE:ON CyclomaticComplexity
-
- /**
- * Helper method to get parent shardIds of the current shard - includes the parent shardIds if:
- * a/ they are not null
- * b/ if they exist in the current shard map (i.e. haven't expired)
- *
- * @param shard Will return parents of this shard
- * @param shardIdToShardMapOfAllKinesisShards ShardId->Shard map containing all shards obtained via DescribeStream.
- * @return Set of parentShardIds
- */
- static Set getParentShardIds(final Shard shard,
- final Map shardIdToShardMapOfAllKinesisShards) {
- final Set parentShardIds = new HashSet<>(2);
- final String parentShardId = shard.parentShardId();
- if (parentShardId != null && shardIdToShardMapOfAllKinesisShards.containsKey(parentShardId)) {
- parentShardIds.add(parentShardId);
- }
- final String adjacentParentShardId = shard.adjacentParentShardId();
- if (adjacentParentShardId != null && shardIdToShardMapOfAllKinesisShards.containsKey(adjacentParentShardId)) {
- parentShardIds.add(adjacentParentShardId);
- }
- return parentShardIds;
- }
-
- /**
- * Delete leases corresponding to shards that no longer exist in the stream. Current scheme: Delete a lease if:
- *
- * - The corresponding shard is not present in the list of Kinesis shards
- * - The parentShardIds listed in the lease are also not present in the list of Kinesis shards.
- *
- *
- * @param shards
- * List of all Kinesis shards (assumed to be a consistent snapshot - when stream is in Active state).
- * @param trackedLeases
- * List of
- * @param leaseRefresher
- * @throws KinesisClientLibIOException
- * Thrown if we couldn't get a fresh shard list from Kinesis.
- * @throws ProvisionedThroughputException
- * @throws InvalidStateException
- * @throws DependencyException
- */
- private static void cleanupGarbageLeases(@NonNull final ShardDetector shardDetector, final List shards,
- final List trackedLeases, final LeaseRefresher leaseRefresher) throws KinesisClientLibIOException,
- DependencyException, InvalidStateException, ProvisionedThroughputException {
- final Set kinesisShards = shards.stream().map(Shard::shardId).collect(Collectors.toSet());
-
- // Check if there are leases for non-existent shards
- final List garbageLeases = trackedLeases.stream()
- .filter(lease -> isCandidateForCleanup(lease, kinesisShards)).collect(Collectors.toList());
-
- if (!CollectionUtils.isNullOrEmpty(garbageLeases)) {
- log.info("Found {} candidate leases for cleanup. Refreshing list of"
- + " Kinesis shards to pick up recent/latest shards", garbageLeases.size());
- final Set currentKinesisShardIds = getShardList(shardDetector).stream().map(Shard::shardId)
- .collect(Collectors.toSet());
-
- for (Lease lease : garbageLeases) {
- if (isCandidateForCleanup(lease, currentKinesisShardIds)) {
- log.info("Deleting lease for shard {} as it is not present in Kinesis stream.", lease.leaseKey());
- leaseRefresher.deleteLease(lease);
- }
- }
- }
- }
-
- /**
- * Note: This method has package level access, solely for testing purposes.
- *
- * @param lease Candidate shard we are considering for deletion.
- * @param currentKinesisShardIds
- * @return true if neither the shard (corresponding to the lease), nor its parents are present in
- * currentKinesisShardIds
- * @throws KinesisClientLibIOException Thrown if currentKinesisShardIds contains a parent shard but not the child
- * shard (we are evaluating for deletion).
- */
- static boolean isCandidateForCleanup(final Lease lease, final Set currentKinesisShardIds)
- throws KinesisClientLibIOException {
- boolean isCandidateForCleanup = true;
-
- if (currentKinesisShardIds.contains(lease.leaseKey())) {
- isCandidateForCleanup = false;
- } else {
- log.info("Found lease for non-existent shard: {}. Checking its parent shards", lease.leaseKey());
- final Set parentShardIds = lease.parentShardIds();
- for (String parentShardId : parentShardIds) {
-
- // Throw an exception if the parent shard exists (but the child does not).
- // This may be a (rare) race condition between fetching the shard list and Kinesis expiring shards.
- if (currentKinesisShardIds.contains(parentShardId)) {
- final String message = String.format("Parent shard %s exists but not the child shard %s",
- parentShardId, lease.leaseKey());
- log.info(message);
- throw new KinesisClientLibIOException(message);
- }
- }
- }
-
- return isCandidateForCleanup;
- }
-
- /**
- * Private helper method.
- * Clean up leases for shards that meet the following criteria:
- * a/ the shard has been fully processed (checkpoint is set to SHARD_END)
- * b/ we've begun processing all the child shards: we have leases for all child shards and their checkpoint is not
- * TRIM_HORIZON.
- *
- * @param currentLeases List of leases we evaluate for clean up
- * @param shardIdToShardMap Map of shardId->Shard (assumed to include all Kinesis shards)
- * @param shardIdToChildShardIdsMap Map of shardId->childShardIds (assumed to include all Kinesis shards)
- * @param trackedLeases List of all leases we are tracking.
- * @param leaseRefresher Lease refresher (will be used to delete leases)
- * @throws DependencyException
- * @throws InvalidStateException
- * @throws ProvisionedThroughputException
- * @throws KinesisClientLibIOException
- */
- private synchronized void cleanupLeasesOfFinishedShards(final Collection currentLeases,
- final Map shardIdToShardMap, final Map> shardIdToChildShardIdsMap,
- final List trackedLeases, final LeaseRefresher leaseRefresher) throws DependencyException,
- InvalidStateException, ProvisionedThroughputException, KinesisClientLibIOException {
- final List leasesOfClosedShards = currentLeases.stream()
- .filter(lease -> lease.checkpoint().equals(ExtendedSequenceNumber.SHARD_END))
- .collect(Collectors.toList());
- final Set shardIdsOfClosedShards = leasesOfClosedShards.stream().map(Lease::leaseKey)
- .collect(Collectors.toSet());
-
- if (!CollectionUtils.isNullOrEmpty(leasesOfClosedShards)) {
- assertClosedShardsAreCoveredOrAbsent(shardIdToShardMap, shardIdToChildShardIdsMap, shardIdsOfClosedShards);
- Comparator startingSequenceNumberComparator = new StartingSequenceNumberAndShardIdBasedComparator(
- shardIdToShardMap);
- leasesOfClosedShards.sort(startingSequenceNumberComparator);
- final Map trackedLeaseMap = trackedLeases.stream()
- .collect(Collectors.toMap(Lease::leaseKey, Function.identity()));
-
- for (Lease leaseOfClosedShard : leasesOfClosedShards) {
- final String closedShardId = leaseOfClosedShard.leaseKey();
- final Set childShardIds = shardIdToChildShardIdsMap.get(closedShardId);
- if (closedShardId != null && !CollectionUtils.isNullOrEmpty(childShardIds)) {
- cleanupLeaseForClosedShard(closedShardId, childShardIds, trackedLeaseMap, leaseRefresher);
- }
- }
- }
- }
-
- /**
- * Delete lease for the closed shard. Rules for deletion are:
- * a/ the checkpoint for the closed shard is SHARD_END,
- * b/ there are leases for all the childShardIds and their checkpoint is NOT TRIM_HORIZON
- * Note: This method has package level access solely for testing purposes.
- *
- * @param closedShardId Identifies the closed shard
- * @param childShardIds ShardIds of children of the closed shard
- * @param trackedLeases shardId->Lease map with all leases we are tracking (should not be null)
- * @param leaseRefresher
- * @throws ProvisionedThroughputException
- * @throws InvalidStateException
- * @throws DependencyException
- */
- synchronized void cleanupLeaseForClosedShard(final String closedShardId, final Set childShardIds,
- final Map trackedLeases, final LeaseRefresher leaseRefresher)
- throws DependencyException, InvalidStateException, ProvisionedThroughputException {
- final Lease leaseForClosedShard = trackedLeases.get(closedShardId);
- final List childShardLeases = childShardIds.stream().map(trackedLeases::get).filter(Objects::nonNull)
- .collect(Collectors.toList());
-
- if (leaseForClosedShard != null && leaseForClosedShard.checkpoint().equals(ExtendedSequenceNumber.SHARD_END)
- && childShardLeases.size() == childShardIds.size()) {
- boolean okayToDelete = true;
- for (Lease lease : childShardLeases) {
- if (lease.checkpoint().equals(ExtendedSequenceNumber.TRIM_HORIZON)) {
- okayToDelete = false;
- break;
- }
- }
-
- if (okayToDelete) {
- log.info("Deleting lease for shard {} as it has been completely processed and processing of child "
- + "shards has begun.", leaseForClosedShard.leaseKey());
- leaseRefresher.deleteLease(leaseForClosedShard);
- }
- }
- }
-
- /**
- * Helper method to create a new Lease POJO for a shard.
- * Note: Package level access only for testing purposes
- *
- * @param shard
- * @return
- */
- private static Lease newKCLLease(final Shard shard) {
- Lease newLease = new Lease();
- newLease.leaseKey(shard.shardId());
- List parentShardIds = new ArrayList<>(2);
- if (shard.parentShardId() != null) {
- parentShardIds.add(shard.parentShardId());
- }
- if (shard.adjacentParentShardId() != null) {
- parentShardIds.add(shard.adjacentParentShardId());
- }
- newLease.parentShardIds(parentShardIds);
- newLease.ownerSwitchesSinceCheckpoint(0L);
-
- return newLease;
- }
-
- /**
- * Helper method to construct a shardId->Shard map for the specified list of shards.
- *
- * @param shards List of shards
- * @return ShardId->Shard map
- */
- static Map constructShardIdToShardMap(final List shards) {
- return shards.stream().collect(Collectors.toMap(Shard::shardId, Function.identity()));
- }
-
- /**
- * Helper method to return all the open shards for a stream.
- * Note: Package level access only for testing purposes.
- *
- * @param allShards All shards returved via DescribeStream. We assume this to represent a consistent shard list.
- * @return List of open shards (shards at the tip of the stream) - may include shards that are not yet active.
- */
- static List getOpenShards(final List allShards) {
- return allShards.stream().filter(shard -> shard.sequenceNumberRange().endingSequenceNumber() == null)
- .peek(shard -> log.debug("Found open shard: {}", shard.shardId())).collect(Collectors.toList());
- }
-
- private static ExtendedSequenceNumber convertToCheckpoint(final InitialPositionInStreamExtended position) {
- ExtendedSequenceNumber checkpoint = null;
-
- if (position.getInitialPositionInStream().equals(InitialPositionInStream.TRIM_HORIZON)) {
- checkpoint = ExtendedSequenceNumber.TRIM_HORIZON;
- } else if (position.getInitialPositionInStream().equals(InitialPositionInStream.LATEST)) {
- checkpoint = ExtendedSequenceNumber.LATEST;
- } else if (position.getInitialPositionInStream().equals(InitialPositionInStream.AT_TIMESTAMP)) {
- checkpoint = ExtendedSequenceNumber.AT_TIMESTAMP;
- }
-
- return checkpoint;
- }
-
- /** Helper class to compare leases based on starting sequence number of the corresponding shards.
- *
- */
- @RequiredArgsConstructor
- private static class StartingSequenceNumberAndShardIdBasedComparator implements Comparator, Serializable {
- private static final long serialVersionUID = 1L;
-
- private final Map shardIdToShardMap;
-
- /**
- * Compares two leases based on the starting sequence number of corresponding shards.
- * If shards are not found in the shardId->shard map supplied, we do a string comparison on the shardIds.
- * We assume that lease1 and lease2 are:
- * a/ not null,
- * b/ shards (if found) have non-null starting sequence numbers
- *
- * {@inheritDoc}
- */
- @Override
- public int compare(final Lease lease1, final Lease lease2) {
- int result = 0;
- final String shardId1 = lease1.leaseKey();
- final String shardId2 = lease2.leaseKey();
- final Shard shard1 = shardIdToShardMap.get(shardId1);
- final Shard shard2 = shardIdToShardMap.get(shardId2);
-
- // If we found shards for the two leases, use comparison of the starting sequence numbers
- if (shard1 != null && shard2 != null) {
- BigInteger sequenceNumber1 = new BigInteger(shard1.sequenceNumberRange().startingSequenceNumber());
- BigInteger sequenceNumber2 = new BigInteger(shard2.sequenceNumberRange().startingSequenceNumber());
- result = sequenceNumber1.compareTo(sequenceNumber2);
- }
-
- if (result == 0) {
- result = shardId1.compareTo(shardId2);
- }
-
- return result;
- }
-
- }
-
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/KinesisShardDetector.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/KinesisShardDetector.java
deleted file mode 100644
index b5645d9d..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/KinesisShardDetector.java
+++ /dev/null
@@ -1,219 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-
-package software.amazon.kinesis.leases;
-
-import java.time.Duration;
-import java.time.Instant;
-import java.time.temporal.ChronoUnit;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-
-import org.apache.commons.lang3.StringUtils;
-
-import lombok.AccessLevel;
-import lombok.Getter;
-import lombok.NonNull;
-import lombok.RequiredArgsConstructor;
-import lombok.Synchronized;
-import lombok.experimental.Accessors;
-import lombok.extern.slf4j.Slf4j;
-import software.amazon.awssdk.services.kinesis.KinesisAsyncClient;
-import software.amazon.awssdk.services.kinesis.model.KinesisException;
-import software.amazon.awssdk.services.kinesis.model.LimitExceededException;
-import software.amazon.awssdk.services.kinesis.model.ListShardsRequest;
-import software.amazon.awssdk.services.kinesis.model.ListShardsResponse;
-import software.amazon.awssdk.services.kinesis.model.ResourceInUseException;
-import software.amazon.awssdk.services.kinesis.model.Shard;
-import software.amazon.awssdk.utils.CollectionUtils;
-import software.amazon.kinesis.annotations.KinesisClientInternalApi;
-import software.amazon.kinesis.common.KinesisRequestsBuilder;
-import software.amazon.kinesis.retrieval.AWSExceptionManager;
-
-/**
- *
- */
-@RequiredArgsConstructor
-@Slf4j
-@Accessors(fluent = true)
-@KinesisClientInternalApi
-public class KinesisShardDetector implements ShardDetector {
- @NonNull
- private final KinesisAsyncClient kinesisClient;
- @NonNull
- private final String streamName;
- private final long listShardsBackoffTimeInMillis;
- private final int maxListShardsRetryAttempts;
- private final long listShardsCacheAllowedAgeInSeconds;
- private final int maxCacheMissesBeforeReload;
- private final int cacheMissWarningModulus;
-
- private volatile Map cachedShardMap = null;
- private volatile Instant lastCacheUpdateTime;
- @Getter(AccessLevel.PACKAGE)
- private AtomicInteger cacheMisses = new AtomicInteger(0);
-
- @Override
- public Shard shard(@NonNull final String shardId) {
- if (CollectionUtils.isNullOrEmpty(this.cachedShardMap)) {
- synchronized (this) {
- if (CollectionUtils.isNullOrEmpty(this.cachedShardMap)) {
- listShards();
- }
- }
- }
-
- Shard shard = cachedShardMap.get(shardId);
-
- if (shard == null) {
- if (cacheMisses.incrementAndGet() > maxCacheMissesBeforeReload || shouldRefreshCache()) {
- synchronized (this) {
- shard = cachedShardMap.get(shardId);
-
- if (shard == null) {
- log.info("Too many shard map cache misses or cache is out of date -- forcing a refresh");
- listShards();
- shard = cachedShardMap.get(shardId);
-
- if (shard == null) {
- log.warn("Even after cache refresh shard '{}' wasn't found. This could indicate a bigger"
- + " problem.", shardId);
- }
-
- cacheMisses.set(0);
- } else {
- //
- // If the shardmap got updated, go ahead and set cache misses to 0
- //
- cacheMisses.set(0);
- }
- }
- }
- }
-
- if (shard == null) {
- final String message = String.format("Cannot find the shard given the shardId %s. Cache misses: %s",
- shardId, cacheMisses);
- if (cacheMisses.get() % cacheMissWarningModulus == 0) {
- log.warn(message);
- } else {
- log.debug(message);
- }
- }
-
- return shard;
- }
-
- @Override
- @Synchronized
- public List listShards() {
- final List shards = new ArrayList<>();
- ListShardsResponse result;
- String nextToken = null;
-
- do {
- result = listShards(nextToken);
-
- if (result == null) {
- /*
- * If listShards ever returns null, we should bail and return null. This indicates the stream is not
- * in ACTIVE or UPDATING state and we may not have accurate/consistent information about the stream.
- */
- return null;
- } else {
- shards.addAll(result.shards());
- nextToken = result.nextToken();
- }
- } while (StringUtils.isNotEmpty(result.nextToken()));
-
- cachedShardMap(shards);
- return shards;
- }
-
- private ListShardsResponse listShards(final String nextToken) {
- final AWSExceptionManager exceptionManager = new AWSExceptionManager();
- exceptionManager.add(LimitExceededException.class, t -> t);
- exceptionManager.add(ResourceInUseException.class, t -> t);
- exceptionManager.add(KinesisException.class, t -> t);
-
- ListShardsRequest.Builder request = KinesisRequestsBuilder.listShardsRequestBuilder();
- if (StringUtils.isEmpty(nextToken)) {
- request = request.streamName(streamName);
- } else {
- request = request.nextToken(nextToken);
- }
- ListShardsResponse result = null;
- LimitExceededException lastException = null;
- int remainingRetries = maxListShardsRetryAttempts;
-
- while (result == null) {
-
- try {
- try {
- result = kinesisClient.listShards(request.build()).get();
- } catch (ExecutionException e) {
- throw exceptionManager.apply(e.getCause());
- } catch (InterruptedException e) {
- // TODO: check if this is the correct behavior for Interrupted Exception
- log.debug("Interrupted exception caught, shutdown initiated, returning null");
- return null;
- }
- } catch (ResourceInUseException e) {
- log.info("Stream is not in Active/Updating status, returning null (wait until stream is in"
- + " Active or Updating)");
- return null;
- } catch (LimitExceededException e) {
- log.info("Got LimitExceededException when listing shards {}. Backing off for {} millis.", streamName,
- listShardsBackoffTimeInMillis);
- try {
- Thread.sleep(listShardsBackoffTimeInMillis);
- } catch (InterruptedException ie) {
- log.debug("Stream {} : Sleep was interrupted ", streamName, ie);
- }
- lastException = e;
- }
- remainingRetries--;
- if (remainingRetries <= 0 && result == null) {
- if (lastException != null) {
- throw lastException;
- }
- throw new IllegalStateException("Received null from ListShards call.");
- }
- }
- return result;
- }
-
- void cachedShardMap(final List shards) {
- cachedShardMap = shards.stream().collect(Collectors.toMap(Shard::shardId, Function.identity()));
- lastCacheUpdateTime = Instant.now();
- }
-
- private boolean shouldRefreshCache() {
- final Duration secondsSinceLastUpdate = Duration.between(lastCacheUpdateTime, Instant.now());
- final String message = String.format("Shard map cache is %d seconds old", secondsSinceLastUpdate.getSeconds());
- if (secondsSinceLastUpdate.compareTo(Duration.of(listShardsCacheAllowedAgeInSeconds, ChronoUnit.SECONDS)) > 0) {
- log.info("{}. Age exceeds limit of {} seconds -- Refreshing.", message, listShardsCacheAllowedAgeInSeconds);
- return true;
- }
-
- log.debug("{}. Age doesn't exceed limit of {} seconds.", message, listShardsCacheAllowedAgeInSeconds);
- return false;
- }
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseCoordinator.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseCoordinator.java
deleted file mode 100644
index 482555b3..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseCoordinator.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-
-package software.amazon.kinesis.leases;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.UUID;
-
-import software.amazon.kinesis.leases.dynamodb.DynamoDBLeaseCoordinator;
-import software.amazon.kinesis.leases.exceptions.DependencyException;
-import software.amazon.kinesis.leases.exceptions.InvalidStateException;
-import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException;
-
-/**
- *
- */
-public interface LeaseCoordinator {
- /**
- * Initialize the lease coordinator (create the lease table if needed).
- * @throws DependencyException
- * @throws ProvisionedThroughputException
- */
- void initialize() throws ProvisionedThroughputException, DependencyException, IllegalStateException;
-
- /**
- * Start background LeaseHolder and LeaseTaker threads.
- * @throws ProvisionedThroughputException If we can't talk to DynamoDB due to insufficient capacity.
- * @throws InvalidStateException If the lease table doesn't exist
- * @throws DependencyException If we encountered exception taking to DynamoDB
- */
- void start() throws DependencyException, InvalidStateException, ProvisionedThroughputException;
-
- /**
- * Runs a single iteration of the lease taker - used by integration tests.
- *
- * @throws InvalidStateException
- * @throws DependencyException
- */
- void runLeaseTaker() throws DependencyException, InvalidStateException;
-
- /**
- * Runs a single iteration of the lease renewer - used by integration tests.
- *
- * @throws InvalidStateException
- * @throws DependencyException
- */
- void runLeaseRenewer() throws DependencyException, InvalidStateException;
-
- /**
- * @return true if this LeaseCoordinator is running
- */
- boolean isRunning();
-
- /**
- * @return workerIdentifier
- */
- String workerIdentifier();
-
- /**
- * @return {@link LeaseRefresher}
- */
- LeaseRefresher leaseRefresher();
-
- /**
- * @return currently held leases
- */
- Collection getAssignments();
-
- /**
- * @param leaseKey lease key to fetch currently held lease for
- *
- * @return deep copy of currently held Lease for given key, or null if we don't hold the lease for that key
- */
- Lease getCurrentlyHeldLease(String leaseKey);
-
- /**
- * Updates application-specific lease values in DynamoDB.
- *
- * @param lease lease object containing updated values
- * @param concurrencyToken obtained by calling Lease.concurrencyToken for a currently held lease
- *
- * @return true if update succeeded, false otherwise
- *
- * @throws InvalidStateException if lease table does not exist
- * @throws ProvisionedThroughputException if DynamoDB update fails due to lack of capacity
- * @throws DependencyException if DynamoDB update fails in an unexpected way
- */
- boolean updateLease(Lease lease, UUID concurrencyToken, String operation, String shardId)
- throws DependencyException, InvalidStateException, ProvisionedThroughputException;
-
- /**
- * Requests the cancellation of the lease taker.
- */
- void stopLeaseTaker();
-
- /**
- * Requests that renewals for the given lease are stopped.
- *
- * @param lease the lease to stop renewing.
- */
- void dropLease(Lease lease);
-
- /**
- * Stops background threads and waits for specific amount of time for all background tasks to complete.
- * If tasks are not completed after this time, method will shutdown thread pool forcefully and return.
- */
- void stop();
-
- /**
- * @return Current shard/lease assignments
- */
- List getCurrentAssignments();
-
- /**
- * Default implementation returns an empty list and concrete implementation is expected to return all leases
- * for the application that are in the lease table. This enables application managing Kcl Scheduler to take care of
- * horizontal scaling for example.
- *
- * @return all leases for the application that are in the lease table
- */
- default List allLeases() {
- return Collections.emptyList();
- }
-
- /**
- * @param writeCapacity The DynamoDB table used for tracking leases will be provisioned with the specified initial
- * write capacity
- * @return LeaseCoordinator
- */
- DynamoDBLeaseCoordinator initialLeaseTableWriteCapacity(long writeCapacity);
-
- /**
- * @param readCapacity The DynamoDB table used for tracking leases will be provisioned with the specified initial
- * read capacity
- * @return LeaseCoordinator
- */
- DynamoDBLeaseCoordinator initialLeaseTableReadCapacity(long readCapacity);
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseManagementConfig.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseManagementConfig.java
deleted file mode 100644
index 5c98bae6..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseManagementConfig.java
+++ /dev/null
@@ -1,269 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-
-package software.amazon.kinesis.leases;
-
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-
-import lombok.Data;
-import lombok.NonNull;
-import lombok.experimental.Accessors;
-import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
-import software.amazon.awssdk.services.kinesis.KinesisAsyncClient;
-import software.amazon.kinesis.common.InitialPositionInStream;
-import software.amazon.kinesis.common.InitialPositionInStreamExtended;
-import software.amazon.kinesis.leases.dynamodb.DynamoDBLeaseManagementFactory;
-import software.amazon.kinesis.leases.dynamodb.TableCreatorCallback;
-import software.amazon.kinesis.metrics.MetricsFactory;
-import software.amazon.kinesis.metrics.NullMetricsFactory;
-
-/**
- * Used by the KCL to configure lease management.
- */
-@Data
-@Accessors(fluent = true)
-public class LeaseManagementConfig {
- /**
- * Name of the table to use in DynamoDB
- *
- * @return String
- */
- @NonNull
- private final String tableName;
- /**
- * Client to be used to access DynamoDB service.
- *
- * @return {@link DynamoDbAsyncClient}
- */
- @NonNull
- private final DynamoDbAsyncClient dynamoDBClient;
- /**
- * Client to be used to access Kinesis Data Streams service.
- *
- * @return {@link KinesisAsyncClient}
- */
- @NonNull
- private final KinesisAsyncClient kinesisClient;
- /**
- * Name of the Kinesis Data Stream to read records from.
- */
- @NonNull
- private final String streamName;
- /**
- * Used to distinguish different workers/processes of a KCL application.
- *
- * @return String
- */
- @NonNull
- private final String workerIdentifier;
-
- /**
- * Fail over time in milliseconds. A worker which does not renew it's lease within this time interval
- * will be regarded as having problems and it's shards will be assigned to other workers.
- * For applications that have a large number of shards, this may be set to a higher number to reduce
- * the number of DynamoDB IOPS required for tracking leases.
- *
- * Default value: 10000L
- */
- private long failoverTimeMillis = 10000L;
-
- /**
- * Shard sync interval in milliseconds - e.g. wait for this long between shard sync tasks.
- *
- * Default value: 60000L
- */
- private long shardSyncIntervalMillis = 60000L;
-
- /**
- * Cleanup leases upon shards completion (don't wait until they expire in Kinesis).
- * Keeping leases takes some tracking/resources (e.g. they need to be renewed, assigned), so by default we try
- * to delete the ones we don't need any longer.
- *
- * Default value: true
- */
- private boolean cleanupLeasesUponShardCompletion = true;
-
- /**
- * The max number of leases (shards) this worker should process.
- * This can be useful to avoid overloading (and thrashing) a worker when a host has resource constraints
- * or during deployment.
- *
- * NOTE: Setting this to a low value can cause data loss if workers are not able to pick up all shards in the
- * stream due to the max limit.
- *
- * Default value: {@link Integer#MAX_VALUE}
- */
- private int maxLeasesForWorker = Integer.MAX_VALUE;;
-
- /**
- * Max leases to steal from another worker at one time (for load balancing).
- * Setting this to a higher number can allow for faster load convergence (e.g. during deployments, cold starts),
- * but can cause higher churn in the system.
- *
- * Default value: 1
- */
- private int maxLeasesToStealAtOneTime = 1;
-
- /**
- * The Amazon DynamoDB table used for tracking leases will be provisioned with this read capacity.
- *
- * Default value: 10
- */
- private int initialLeaseTableReadCapacity = 10;
-
- /**
- * The Amazon DynamoDB table used for tracking leases will be provisioned with this write capacity.
- *
- * Default value: 10
- */
- private int initialLeaseTableWriteCapacity = 10;
-
- /**
- * The size of the thread pool to create for the lease renewer to use.
- *
- * Default value: 20
- */
- private int maxLeaseRenewalThreads = 20;
-
- /**
- *
- */
- private boolean ignoreUnexpectedChildShards = false;
-
- /**
- *
- */
- private boolean consistentReads = false;
-
- private long listShardsBackoffTimeInMillis = 1500L;
-
- private int maxListShardsRetryAttempts = 50;
-
- public long epsilonMillis = 25L;
-
- /**
- * The initial position for getting records from Kinesis streams.
- *
- * Default value: {@link InitialPositionInStream#TRIM_HORIZON}
- */
- private InitialPositionInStreamExtended initialPositionInStream =
- InitialPositionInStreamExtended.newInitialPosition(InitialPositionInStream.TRIM_HORIZON);
-
- private int maxCacheMissesBeforeReload = 1000;
- private long listShardsCacheAllowedAgeInSeconds = 30;
- private int cacheMissWarningModulus = 250;
-
- private MetricsFactory metricsFactory = new NullMetricsFactory();
-
- /**
- * Returns the metrics factory.
- *
- *
- * NOTE: This method is deprecated and will be removed in a future release. This metrics factory is not being used
- * in the KCL.
- *
- *
- * @return
- */
- @Deprecated
- public MetricsFactory metricsFactory() {
- return metricsFactory;
- }
-
- /**
- * Sets the metrics factory.
- *
- *
- * NOTE: This method is deprecated and will be removed in a future release. This metrics factory is not being used
- * in the KCL.
- *
- *
- * @param metricsFactory
- */
- @Deprecated
- public LeaseManagementConfig metricsFactory(final MetricsFactory metricsFactory) {
- this.metricsFactory = metricsFactory;
- return this;
- }
-
- /**
- * The {@link ExecutorService} to be used by {@link ShardSyncTaskManager}.
- *
- * Default value: {@link LeaseManagementThreadPool}
- */
- private ExecutorService executorService = new LeaseManagementThreadPool(
- new ThreadFactoryBuilder().setNameFormat("ShardSyncTaskManager-%04d").build());
-
- static class LeaseManagementThreadPool extends ThreadPoolExecutor {
- private static final long DEFAULT_KEEP_ALIVE_TIME = 60L;
-
- LeaseManagementThreadPool(ThreadFactory threadFactory) {
- super(0, Integer.MAX_VALUE, DEFAULT_KEEP_ALIVE_TIME, TimeUnit.SECONDS, new SynchronousQueue<>(),
- threadFactory);
- }
- }
-
- /**
- * Callback used with DynamoDB lease management. Callback is invoked once the table is newly created and is in the
- * active status.
- *
- *
- * Default value: {@link TableCreatorCallback#NOOP_TABLE_CREATOR_CALLBACK}
- *
- */
- private TableCreatorCallback tableCreatorCallback = TableCreatorCallback.NOOP_TABLE_CREATOR_CALLBACK;
-
- private HierarchicalShardSyncer hierarchicalShardSyncer = new HierarchicalShardSyncer();
-
- private LeaseManagementFactory leaseManagementFactory;
-
- public LeaseManagementFactory leaseManagementFactory() {
- if (leaseManagementFactory == null) {
- leaseManagementFactory = new DynamoDBLeaseManagementFactory(kinesisClient(),
- streamName(),
- dynamoDBClient(),
- tableName(),
- workerIdentifier(),
- executorService(),
- initialPositionInStream(),
- failoverTimeMillis(),
- epsilonMillis(),
- maxLeasesForWorker(),
- maxLeasesToStealAtOneTime(),
- maxLeaseRenewalThreads(),
- cleanupLeasesUponShardCompletion(),
- ignoreUnexpectedChildShards(),
- shardSyncIntervalMillis(),
- consistentReads(),
- listShardsBackoffTimeInMillis(),
- maxListShardsRetryAttempts(),
- maxCacheMissesBeforeReload(),
- listShardsCacheAllowedAgeInSeconds(),
- cacheMissWarningModulus(),
- initialLeaseTableReadCapacity(),
- initialLeaseTableWriteCapacity(),
- hierarchicalShardSyncer(),
- tableCreatorCallback());
- }
- return leaseManagementFactory;
- }
-
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseManagementFactory.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseManagementFactory.java
deleted file mode 100644
index 5e685d31..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseManagementFactory.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-
-package software.amazon.kinesis.leases;
-
-import software.amazon.kinesis.leases.dynamodb.DynamoDBLeaseRefresher;
-import software.amazon.kinesis.metrics.MetricsFactory;
-
-/**
- *
- */
-public interface LeaseManagementFactory {
- LeaseCoordinator createLeaseCoordinator(MetricsFactory metricsFactory);
-
- ShardSyncTaskManager createShardSyncTaskManager(MetricsFactory metricsFactory);
-
- DynamoDBLeaseRefresher createLeaseRefresher();
-
- ShardDetector createShardDetector();
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseSerializer.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseSerializer.java
deleted file mode 100644
index e4d8f6f3..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseSerializer.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-package software.amazon.kinesis.leases;
-
-import java.util.Collection;
-import java.util.Map;
-
-
-import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
-import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
-import software.amazon.awssdk.services.dynamodb.model.AttributeValueUpdate;
-import software.amazon.awssdk.services.dynamodb.model.ExpectedAttributeValue;
-import software.amazon.awssdk.services.dynamodb.model.KeySchemaElement;
-import software.amazon.kinesis.leases.Lease;
-
-/**
- * Utility class that manages the mapping of Lease objects/operations to records in DynamoDB.
- */
-public interface LeaseSerializer {
-
- /**
- * Construct a DynamoDB record out of a Lease object
- *
- * @param lease lease object to serialize
- * @return an attribute value map representing the lease object
- */
- Map toDynamoRecord(Lease lease);
-
- /**
- * Construct a Lease object out of a DynamoDB record.
- *
- * @param dynamoRecord attribute value map from DynamoDB
- * @return a deserialized lease object representing the attribute value map
- */
- Lease fromDynamoRecord(Map dynamoRecord);
-
- /**
- * @param lease
- * @return the attribute value map representing a Lease's hash key given a Lease object.
- */
- Map getDynamoHashKey(Lease lease);
-
- /**
- * Special getDynamoHashKey implementation used by {@link LeaseRefresher#getLease(String)}.
- *
- * @param leaseKey
- * @return the attribute value map representing a Lease's hash key given a string.
- */
- Map getDynamoHashKey(String leaseKey);
-
- /**
- * @param lease
- * @return the attribute value map asserting that a lease counter is what we expect.
- */
- Map getDynamoLeaseCounterExpectation(Lease lease);
-
- /**
- * @param lease
- * @return the attribute value map asserting that the lease owner is what we expect.
- */
- Map getDynamoLeaseOwnerExpectation(Lease lease);
-
- /**
- * @return the attribute value map asserting that a lease does not exist.
- */
- Map getDynamoNonexistantExpectation();
-
- /**
- * @param lease
- * @return the attribute value map that increments a lease counter
- */
- Map getDynamoLeaseCounterUpdate(Lease lease);
-
- /**
- * @param lease
- * @param newOwner
- * @return the attribute value map that takes a lease for a new owner
- */
- Map getDynamoTakeLeaseUpdate(Lease lease, String newOwner);
-
- /**
- * @param lease
- * @return the attribute value map that voids a lease
- */
- Map getDynamoEvictLeaseUpdate(Lease lease);
-
- /**
- * @param lease
- * @return the attribute value map that updates application-specific data for a lease and increments the lease
- * counter
- */
- Map getDynamoUpdateLeaseUpdate(Lease lease);
-
- /**
- * @return the key schema for creating a DynamoDB table to store leases
- */
- Collection getKeySchema();
-
- /**
- * @return attribute definitions for creating a DynamoDB table to store leases
- */
- Collection getAttributeDefinitions();
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseTaker.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseTaker.java
deleted file mode 100644
index 394375b3..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseTaker.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-package software.amazon.kinesis.leases;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-import software.amazon.kinesis.leases.exceptions.DependencyException;
-import software.amazon.kinesis.leases.exceptions.InvalidStateException;
-
-/**
- * ILeaseTaker is used by LeaseCoordinator to take new leases, or leases that other workers fail to renew. Each
- * LeaseCoordinator instance corresponds to one worker and uses exactly one ILeaseTaker to take leases for that worker.
- */
-public interface LeaseTaker {
-
- /**
- * Compute the set of leases available to be taken and attempt to take them. Lease taking rules are:
- *
- * 1) If a lease's counter hasn't changed in long enough, try to take it.
- * 2) If we see a lease we've never seen before, take it only if owner == null. If it's owned, odds are the owner is
- * holding it. We can't tell until we see it more than once.
- * 3) For load balancing purposes, you may violate rules 1 and 2 for EXACTLY ONE lease per call of takeLeases().
- *
- * @return map of shardId to Lease object for leases we just successfully took.
- *
- * @throws DependencyException on unexpected DynamoDB failures
- * @throws InvalidStateException if lease table does not exist
- */
- Map takeLeases() throws DependencyException, InvalidStateException;
-
- /**
- * @return workerIdentifier for this LeaseTaker
- */
- String getWorkerIdentifier();
-
- /**
- * Default implementation returns an empty list and concrete implementaion is expected to return all leases
- * for the application that are in the lease table either by reading lease table or from an internal cache.
- *
- * @return all leases for the application that are in the lease table
- */
- default List allLeases() {
- return Collections.emptyList();
- }
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/NoOpShardPrioritization.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/NoOpShardPrioritization.java
deleted file mode 100644
index ec93d764..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/NoOpShardPrioritization.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-package software.amazon.kinesis.leases;
-
-import java.util.List;
-
-/**
- * Shard Prioritization that returns the same original list of shards without any modifications.
- */
-public class NoOpShardPrioritization implements
- ShardPrioritization {
-
- /**
- * Empty constructor for NoOp Shard Prioritization.
- */
- public NoOpShardPrioritization() {
- }
-
- @Override
- public List prioritize(List original) {
- return original;
- }
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ShardDetector.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ShardDetector.java
deleted file mode 100644
index ebcb190a..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ShardDetector.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-
-package software.amazon.kinesis.leases;
-
-import software.amazon.awssdk.services.kinesis.model.Shard;
-
-import java.util.List;
-
-/**
- *
- */
-public interface ShardDetector {
- Shard shard(String shardId);
-
- List listShards();
-
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ShardPrioritization.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ShardPrioritization.java
deleted file mode 100644
index 11b7586a..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ShardPrioritization.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-package software.amazon.kinesis.leases;
-
-import java.util.List;
-
-/**
- * Provides logic to prioritize or filter shards before their execution.
- */
-public interface ShardPrioritization {
-
- /**
- * Returns new list of shards ordered based on their priority.
- * Resulted list may have fewer shards compared to original list
- *
- * @param original
- * list of shards needed to be prioritized
- * @return new list that contains only shards that should be processed
- */
- List prioritize(List original);
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ShardSyncTask.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ShardSyncTask.java
deleted file mode 100644
index 046efdea..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ShardSyncTask.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-package software.amazon.kinesis.leases;
-
-import lombok.NonNull;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import software.amazon.kinesis.annotations.KinesisClientInternalApi;
-import software.amazon.kinesis.common.InitialPositionInStreamExtended;
-import software.amazon.kinesis.lifecycle.ConsumerTask;
-import software.amazon.kinesis.lifecycle.TaskResult;
-import software.amazon.kinesis.lifecycle.TaskType;
-import software.amazon.kinesis.metrics.MetricsFactory;
-import software.amazon.kinesis.metrics.MetricsScope;
-import software.amazon.kinesis.metrics.MetricsUtil;
-
-/**
- * This task syncs leases/activies with shards of the stream.
- * It will create new leases/activites when it discovers new shards (e.g. setup/resharding).
- * It will clean up leases/activities for shards that have been completely processed (if
- * cleanupLeasesUponShardCompletion is true).
- */
-@RequiredArgsConstructor
-@Slf4j
-@KinesisClientInternalApi
-public class ShardSyncTask implements ConsumerTask {
- private final String SHARD_SYNC_TASK_OPERATION = "ShardSyncTask";
-
- @NonNull
- private final ShardDetector shardDetector;
- @NonNull
- private final LeaseRefresher leaseRefresher;
- @NonNull
- private final InitialPositionInStreamExtended initialPosition;
- private final boolean cleanupLeasesUponShardCompletion;
- private final boolean ignoreUnexpectedChildShards;
- private final long shardSyncTaskIdleTimeMillis;
- @NonNull
- private final HierarchicalShardSyncer hierarchicalShardSyncer;
- @NonNull
- private final MetricsFactory metricsFactory;
-
- private final TaskType taskType = TaskType.SHARDSYNC;
-
- /*
- * (non-Javadoc)
- * @see com.amazonaws.services.kinesis.clientlibrary.lib.worker.ConsumerTask#call()
- */
- @Override
- public TaskResult call() {
- Exception exception = null;
- final MetricsScope scope = MetricsUtil.createMetricsWithOperation(metricsFactory, SHARD_SYNC_TASK_OPERATION);
-
- try {
- hierarchicalShardSyncer.checkAndCreateLeaseForNewShards(shardDetector, leaseRefresher, initialPosition,
- cleanupLeasesUponShardCompletion, ignoreUnexpectedChildShards, scope);
- if (shardSyncTaskIdleTimeMillis > 0) {
- Thread.sleep(shardSyncTaskIdleTimeMillis);
- }
- } catch (Exception e) {
- log.error("Caught exception while sync'ing Kinesis shards and leases", e);
- exception = e;
- } finally {
- MetricsUtil.endScope(scope);
- }
-
- return new TaskResult(exception);
- }
-
- /*
- * (non-Javadoc)
- * @see com.amazonaws.services.kinesis.clientlibrary.lib.worker.ConsumerTask#taskType()
- */
- @Override
- public TaskType taskType() {
- return taskType;
- }
-
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ShardSyncTaskManager.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ShardSyncTaskManager.java
deleted file mode 100644
index d97c9b90..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ShardSyncTaskManager.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-package software.amazon.kinesis.leases;
-
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-
-import software.amazon.kinesis.common.InitialPositionInStreamExtended;
-
-import lombok.Data;
-import lombok.NonNull;
-import lombok.experimental.Accessors;
-import lombok.extern.slf4j.Slf4j;
-import software.amazon.kinesis.lifecycle.ConsumerTask;
-import software.amazon.kinesis.lifecycle.TaskResult;
-import software.amazon.kinesis.metrics.MetricsFactory;
-import software.amazon.kinesis.metrics.MetricsCollectingTaskDecorator;
-
-/**
- * The ShardSyncTaskManager is used to track the task to sync shards with leases (create leases for new
- * Kinesis shards, remove obsolete leases). We'll have at most one outstanding sync task at any time.
- * Worker will use this class to kick off a sync task when it finds shards which have been completely processed.
- */
-@Data
-@Accessors(fluent = true)
-@Slf4j
-public class ShardSyncTaskManager {
- @NonNull
- private final ShardDetector shardDetector;
- @NonNull
- private final LeaseRefresher leaseRefresher;
- @NonNull
- private final InitialPositionInStreamExtended initialPositionInStream;
- private final boolean cleanupLeasesUponShardCompletion;
- private final boolean ignoreUnexpectedChildShards;
- private final long shardSyncIdleTimeMillis;
- @NonNull
- private final ExecutorService executorService;
- @NonNull
- private final HierarchicalShardSyncer hierarchicalShardSyncer;
- @NonNull
- private final MetricsFactory metricsFactory;
-
- /**
- * Constructor.
- *
- * NOTE: This constructor is deprecated and will be removed in a future release.
- *
- * @param shardDetector
- * @param leaseRefresher
- * @param initialPositionInStream
- * @param cleanupLeasesUponShardCompletion
- * @param ignoreUnexpectedChildShards
- * @param shardSyncIdleTimeMillis
- * @param executorService
- * @param metricsFactory
- */
- @Deprecated
- public ShardSyncTaskManager(ShardDetector shardDetector, LeaseRefresher leaseRefresher,
- InitialPositionInStreamExtended initialPositionInStream, boolean cleanupLeasesUponShardCompletion,
- boolean ignoreUnexpectedChildShards, long shardSyncIdleTimeMillis, ExecutorService executorService,
- MetricsFactory metricsFactory) {
- this.shardDetector = shardDetector;
- this.leaseRefresher = leaseRefresher;
- this.initialPositionInStream = initialPositionInStream;
- this.cleanupLeasesUponShardCompletion = cleanupLeasesUponShardCompletion;
- this.ignoreUnexpectedChildShards = ignoreUnexpectedChildShards;
- this.shardSyncIdleTimeMillis = shardSyncIdleTimeMillis;
- this.executorService = executorService;
- this.hierarchicalShardSyncer = new HierarchicalShardSyncer();
- this.metricsFactory = metricsFactory;
- }
-
- /**
- * Constructor.
- *
- * @param shardDetector
- * @param leaseRefresher
- * @param initialPositionInStream
- * @param cleanupLeasesUponShardCompletion
- * @param ignoreUnexpectedChildShards
- * @param shardSyncIdleTimeMillis
- * @param executorService
- * @param hierarchicalShardSyncer
- * @param metricsFactory
- */
- public ShardSyncTaskManager(ShardDetector shardDetector, LeaseRefresher leaseRefresher,
- InitialPositionInStreamExtended initialPositionInStream, boolean cleanupLeasesUponShardCompletion,
- boolean ignoreUnexpectedChildShards, long shardSyncIdleTimeMillis, ExecutorService executorService,
- HierarchicalShardSyncer hierarchicalShardSyncer, MetricsFactory metricsFactory) {
- this.shardDetector = shardDetector;
- this.leaseRefresher = leaseRefresher;
- this.initialPositionInStream = initialPositionInStream;
- this.cleanupLeasesUponShardCompletion = cleanupLeasesUponShardCompletion;
- this.ignoreUnexpectedChildShards = ignoreUnexpectedChildShards;
- this.shardSyncIdleTimeMillis = shardSyncIdleTimeMillis;
- this.executorService = executorService;
- this.hierarchicalShardSyncer = hierarchicalShardSyncer;
- this.metricsFactory = metricsFactory;
- }
-
- private ConsumerTask currentTask;
- private Future future;
-
- public synchronized boolean syncShardAndLeaseInfo() {
- return checkAndSubmitNextTask();
- }
-
- private synchronized boolean checkAndSubmitNextTask() {
- boolean submittedNewTask = false;
- if ((future == null) || future.isCancelled() || future.isDone()) {
- if ((future != null) && future.isDone()) {
- try {
- TaskResult result = future.get();
- if (result.getException() != null) {
- log.error("Caught exception running {} task: ", currentTask.taskType(),
- result.getException());
- }
- } catch (InterruptedException | ExecutionException e) {
- log.warn("{} task encountered exception.", currentTask.taskType(), e);
- }
- }
-
- currentTask =
- new MetricsCollectingTaskDecorator(
- new ShardSyncTask(shardDetector,
- leaseRefresher,
- initialPositionInStream,
- cleanupLeasesUponShardCompletion,
- ignoreUnexpectedChildShards,
- shardSyncIdleTimeMillis,
- hierarchicalShardSyncer,
- metricsFactory),
- metricsFactory);
- future = executorService.submit(currentTask);
- submittedNewTask = true;
- if (log.isDebugEnabled()) {
- log.debug("Submitted new {} task.", currentTask.taskType());
- }
- } else {
- if (log.isDebugEnabled()) {
- log.debug("Previous {} task still pending. Not submitting new task.", currentTask.taskType());
- }
- }
-
- return submittedNewTask;
- }
-
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseCoordinator.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseCoordinator.java
deleted file mode 100644
index 20a66e9a..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseCoordinator.java
+++ /dev/null
@@ -1,414 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-package software.amazon.kinesis.leases.dynamodb;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedTransferQueue;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-
-import lombok.extern.slf4j.Slf4j;
-import software.amazon.kinesis.annotations.KinesisClientInternalApi;
-import software.amazon.kinesis.leases.Lease;
-import software.amazon.kinesis.leases.LeaseCoordinator;
-import software.amazon.kinesis.leases.LeaseRefresher;
-import software.amazon.kinesis.leases.LeaseRenewer;
-import software.amazon.kinesis.leases.LeaseTaker;
-import software.amazon.kinesis.leases.ShardInfo;
-import software.amazon.kinesis.leases.exceptions.DependencyException;
-import software.amazon.kinesis.leases.exceptions.InvalidStateException;
-import software.amazon.kinesis.leases.exceptions.LeasingException;
-import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException;
-import software.amazon.kinesis.metrics.MetricsFactory;
-import software.amazon.kinesis.metrics.MetricsLevel;
-import software.amazon.kinesis.metrics.MetricsScope;
-import software.amazon.kinesis.metrics.MetricsUtil;
-
-/**
- * LeaseCoordinator abstracts away LeaseTaker and LeaseRenewer from the application code that's using leasing. It owns
- * the scheduling of the two previously mentioned components as well as informing LeaseRenewer when LeaseTaker takes new
- * leases.
- *
- */
-@Slf4j
-@KinesisClientInternalApi
-public class DynamoDBLeaseCoordinator implements LeaseCoordinator {
- // Time to wait for in-flight Runnables to finish when calling .stop();
- private static final long STOP_WAIT_TIME_MILLIS = 2000L;
- private static final ThreadFactory LEASE_COORDINATOR_THREAD_FACTORY = new ThreadFactoryBuilder()
- .setNameFormat("LeaseCoordinator-%04d").setDaemon(true).build();
- private static final ThreadFactory LEASE_RENEWAL_THREAD_FACTORY = new ThreadFactoryBuilder()
- .setNameFormat("LeaseRenewer-%04d").setDaemon(true).build();
-
- private final LeaseRenewer leaseRenewer;
- private final LeaseTaker leaseTaker;
- private final long renewerIntervalMillis;
- private final long takerIntervalMillis;
- private final ExecutorService leaseRenewalThreadpool;
- private final LeaseRefresher leaseRefresher;
- private long initialLeaseTableReadCapacity;
- private long initialLeaseTableWriteCapacity;
- protected final MetricsFactory metricsFactory;
-
- private final Object shutdownLock = new Object();
-
- private ScheduledExecutorService leaseCoordinatorThreadPool;
- private ScheduledFuture takerFuture;
-
- private volatile boolean running = false;
-
- /**
- * Constructor.
- *
- * NOTE: This constructor is deprecated and will be removed in a future release.
- *
- * @param leaseRefresher
- * LeaseRefresher instance to use
- * @param workerIdentifier
- * Identifies the worker (e.g. useful to track lease ownership)
- * @param leaseDurationMillis
- * Duration of a lease
- * @param epsilonMillis
- * Allow for some variance when calculating lease expirations
- * @param maxLeasesForWorker
- * Max leases this Worker can handle at a time
- * @param maxLeasesToStealAtOneTime
- * Steal up to these many leases at a time (for load balancing)
- * @param metricsFactory
- * Used to publish metrics about lease operations
- */
- @Deprecated
- public DynamoDBLeaseCoordinator(final LeaseRefresher leaseRefresher,
- final String workerIdentifier,
- final long leaseDurationMillis,
- final long epsilonMillis,
- final int maxLeasesForWorker,
- final int maxLeasesToStealAtOneTime,
- final int maxLeaseRenewerThreadCount,
- final MetricsFactory metricsFactory) {
- this(leaseRefresher, workerIdentifier, leaseDurationMillis, epsilonMillis, maxLeasesForWorker,
- maxLeasesToStealAtOneTime, maxLeaseRenewerThreadCount,
- TableConstants.DEFAULT_INITIAL_LEASE_TABLE_READ_CAPACITY,
- TableConstants.DEFAULT_INITIAL_LEASE_TABLE_WRITE_CAPACITY, metricsFactory);
- }
-
- /**
- * Constructor.
- *
- * @param leaseRefresher
- * LeaseRefresher instance to use
- * @param workerIdentifier
- * Identifies the worker (e.g. useful to track lease ownership)
- * @param leaseDurationMillis
- * Duration of a lease
- * @param epsilonMillis
- * Allow for some variance when calculating lease expirations
- * @param maxLeasesForWorker
- * Max leases this Worker can handle at a time
- * @param maxLeasesToStealAtOneTime
- * Steal up to these many leases at a time (for load balancing)
- * @param initialLeaseTableReadCapacity
- * Initial dynamodb lease table read iops if creating the lease table
- * @param initialLeaseTableWriteCapacity
- * Initial dynamodb lease table write iops if creating the lease table
- * @param metricsFactory
- * Used to publish metrics about lease operations
- */
- public DynamoDBLeaseCoordinator(final LeaseRefresher leaseRefresher,
- final String workerIdentifier,
- final long leaseDurationMillis,
- final long epsilonMillis,
- final int maxLeasesForWorker,
- final int maxLeasesToStealAtOneTime,
- final int maxLeaseRenewerThreadCount,
- final long initialLeaseTableReadCapacity,
- final long initialLeaseTableWriteCapacity,
- final MetricsFactory metricsFactory) {
- this.leaseRefresher = leaseRefresher;
- this.leaseRenewalThreadpool = getLeaseRenewalExecutorService(maxLeaseRenewerThreadCount);
- this.leaseTaker = new DynamoDBLeaseTaker(leaseRefresher, workerIdentifier, leaseDurationMillis, metricsFactory)
- .withMaxLeasesForWorker(maxLeasesForWorker)
- .withMaxLeasesToStealAtOneTime(maxLeasesToStealAtOneTime);
- this.leaseRenewer = new DynamoDBLeaseRenewer(
- leaseRefresher, workerIdentifier, leaseDurationMillis, leaseRenewalThreadpool, metricsFactory);
- this.renewerIntervalMillis = leaseDurationMillis / 3 - epsilonMillis;
- this.takerIntervalMillis = (leaseDurationMillis + epsilonMillis) * 2;
- if (initialLeaseTableReadCapacity <= 0) {
- throw new IllegalArgumentException("readCapacity should be >= 1");
- }
- this.initialLeaseTableReadCapacity = initialLeaseTableReadCapacity;
- if (initialLeaseTableWriteCapacity <= 0) {
- throw new IllegalArgumentException("writeCapacity should be >= 1");
- }
- this.initialLeaseTableWriteCapacity = initialLeaseTableWriteCapacity;
- this.metricsFactory = metricsFactory;
-
- log.info("With failover time {} ms and epsilon {} ms, LeaseCoordinator will renew leases every {} ms, take"
- + "leases every {} ms, process maximum of {} leases and steal {} lease(s) at a time.",
- leaseDurationMillis,
- epsilonMillis,
- renewerIntervalMillis,
- takerIntervalMillis,
- maxLeasesForWorker,
- maxLeasesToStealAtOneTime);
- }
-
- private class TakerRunnable implements Runnable {
-
- @Override
- public void run() {
- try {
- runLeaseTaker();
- } catch (LeasingException e) {
- log.error("LeasingException encountered in lease taking thread", e);
- } catch (Throwable t) {
- log.error("Throwable encountered in lease taking thread", t);
- }
- }
-
- }
-
- private class RenewerRunnable implements Runnable {
-
- @Override
- public void run() {
- try {
- runLeaseRenewer();
- } catch (LeasingException e) {
- log.error("LeasingException encountered in lease renewing thread", e);
- } catch (Throwable t) {
- log.error("Throwable encountered in lease renewing thread", t);
- }
- }
-
- }
-
- @Override
- public void initialize() throws ProvisionedThroughputException, DependencyException, IllegalStateException {
- final boolean newTableCreated =
- leaseRefresher.createLeaseTableIfNotExists(initialLeaseTableReadCapacity, initialLeaseTableWriteCapacity);
- if (newTableCreated) {
- log.info("Created new lease table for coordinator with initial read capacity of {} and write capacity of {}.",
- initialLeaseTableReadCapacity, initialLeaseTableWriteCapacity);
- }
- // Need to wait for table in active state.
- final long secondsBetweenPolls = 10L;
- final long timeoutSeconds = 600L;
- final boolean isTableActive = leaseRefresher.waitUntilLeaseTableExists(secondsBetweenPolls, timeoutSeconds);
- if (!isTableActive) {
- throw new DependencyException(new IllegalStateException("Creating table timeout"));
- }
- }
-
- @Override
- public void start() throws DependencyException, InvalidStateException, ProvisionedThroughputException {
- leaseRenewer.initialize();
-
- // 2 because we know we'll have at most 2 concurrent tasks at a time.
- leaseCoordinatorThreadPool = Executors.newScheduledThreadPool(2, LEASE_COORDINATOR_THREAD_FACTORY);
-
- // Taker runs with fixed DELAY because we want it to run slower in the event of performance degredation.
- takerFuture = leaseCoordinatorThreadPool.scheduleWithFixedDelay(new TakerRunnable(),
- 0L,
- takerIntervalMillis,
- TimeUnit.MILLISECONDS);
- // Renewer runs at fixed INTERVAL because we want it to run at the same rate in the event of degredation.
- leaseCoordinatorThreadPool.scheduleAtFixedRate(new RenewerRunnable(),
- 0L,
- renewerIntervalMillis,
- TimeUnit.MILLISECONDS);
- running = true;
- }
-
- @Override
- public void runLeaseTaker() throws DependencyException, InvalidStateException {
- MetricsScope scope = MetricsUtil.createMetricsWithOperation(metricsFactory, "TakeLeases");
- long startTime = System.currentTimeMillis();
- boolean success = false;
-
- try {
- Map takenLeases = leaseTaker.takeLeases();
-
- // Only add taken leases to renewer if coordinator is still running.
- synchronized (shutdownLock) {
- if (running) {
- leaseRenewer.addLeasesToRenew(takenLeases.values());
- }
- }
-
- success = true;
- } finally {
- MetricsUtil.addWorkerIdentifier(scope, workerIdentifier());
- MetricsUtil.addSuccessAndLatency(scope, success, startTime, MetricsLevel.SUMMARY);
- MetricsUtil.endScope(scope);
- }
- }
-
- @Override
- public void runLeaseRenewer() throws DependencyException, InvalidStateException {
- leaseRenewer.renewLeases();
- }
-
- @Override
- public Collection getAssignments() {
- return leaseRenewer.getCurrentlyHeldLeases().values();
- }
-
- @Override
- public List allLeases() {
- return leaseTaker.allLeases();
- }
-
- @Override
- public Lease getCurrentlyHeldLease(String leaseKey) {
- return leaseRenewer.getCurrentlyHeldLease(leaseKey);
- }
-
- @Override
- public String workerIdentifier() {
- return leaseTaker.getWorkerIdentifier();
- }
-
- @Override
- public LeaseRefresher leaseRefresher() {
- return leaseRefresher;
- }
-
- @Override
- public void stop() {
- if (leaseCoordinatorThreadPool != null) {
- leaseCoordinatorThreadPool.shutdown();
- try {
- if (leaseCoordinatorThreadPool.awaitTermination(STOP_WAIT_TIME_MILLIS, TimeUnit.MILLISECONDS)) {
- log.info("Worker {} has successfully stopped lease-tracking threads",
- leaseTaker.getWorkerIdentifier());
- } else {
- leaseCoordinatorThreadPool.shutdownNow();
- log.info("Worker {} stopped lease-tracking threads {} ms after stop",
- leaseTaker.getWorkerIdentifier(),
- STOP_WAIT_TIME_MILLIS);
- }
- } catch (InterruptedException e) {
- log.debug("Encountered InterruptedException when awaiting threadpool termination");
- }
- } else {
- log.debug("Threadpool was null, no need to shutdown/terminate threadpool.");
- }
-
- leaseRenewalThreadpool.shutdownNow();
- synchronized (shutdownLock) {
- leaseRenewer.clearCurrentlyHeldLeases();
- running = false;
- }
- }
-
- @Override
- public void stopLeaseTaker() {
- takerFuture.cancel(false);
-
- }
-
- @Override
- public void dropLease(final Lease lease) {
- synchronized (shutdownLock) {
- if (lease != null) {
- leaseRenewer.dropLease(lease);
- }
- }
- }
-
- @Override
- public boolean isRunning() {
- return running;
- }
-
- @Override
- public boolean updateLease(final Lease lease, final UUID concurrencyToken, final String operation,
- final String shardId) throws DependencyException, InvalidStateException, ProvisionedThroughputException {
- return leaseRenewer.updateLease(lease, concurrencyToken, operation, shardId);
- }
-
- /**
- * Returns executor service that should be used for lease renewal.
- * @param maximumPoolSize Maximum allowed thread pool size
- * @return Executor service that should be used for lease renewal.
- */
- private static ExecutorService getLeaseRenewalExecutorService(int maximumPoolSize) {
- int coreLeaseCount = Math.max(maximumPoolSize / 4, 2);
-
- return new ThreadPoolExecutor(coreLeaseCount, maximumPoolSize, 60, TimeUnit.SECONDS,
- new LinkedTransferQueue<>(), LEASE_RENEWAL_THREAD_FACTORY);
- }
-
- @Override
- public List getCurrentAssignments() {
- Collection leases = getAssignments();
- return convertLeasesToAssignments(leases);
- }
-
- private static List convertLeasesToAssignments(final Collection leases) {
- if (leases == null) {
- return Collections.emptyList();
- }
- return leases.stream().map(DynamoDBLeaseCoordinator::convertLeaseToAssignment).collect(Collectors.toList());
- }
-
- public static ShardInfo convertLeaseToAssignment(final Lease lease) {
- return new ShardInfo(lease.leaseKey(), lease.concurrencyToken().toString(), lease.parentShardIds(),
- lease.checkpoint());
- }
-
- /**
- * {@inheritDoc}
- *
- * NOTE: This method is deprecated. Please set the initial capacity through the constructor.
- */
- @Override
- @Deprecated
- public DynamoDBLeaseCoordinator initialLeaseTableReadCapacity(long readCapacity) {
- if (readCapacity <= 0) {
- throw new IllegalArgumentException("readCapacity should be >= 1");
- }
- initialLeaseTableReadCapacity = readCapacity;
- return this;
- }
-
- /**
- * {@inheritDoc}
- *
- * NOTE: This method is deprecated. Please set the initial capacity through the constructor.
- */
- @Override
- @Deprecated
- public DynamoDBLeaseCoordinator initialLeaseTableWriteCapacity(long writeCapacity) {
- if (writeCapacity <= 0) {
- throw new IllegalArgumentException("writeCapacity should be >= 1");
- }
- initialLeaseTableWriteCapacity = writeCapacity;
- return this;
- }
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseManagementFactory.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseManagementFactory.java
deleted file mode 100644
index 1ec3e0b3..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseManagementFactory.java
+++ /dev/null
@@ -1,279 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-
-package software.amazon.kinesis.leases.dynamodb;
-
-import java.util.concurrent.ExecutorService;
-
-import lombok.Data;
-import lombok.NonNull;
-import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
-import software.amazon.awssdk.services.kinesis.KinesisAsyncClient;
-import software.amazon.kinesis.annotations.KinesisClientInternalApi;
-import software.amazon.kinesis.common.InitialPositionInStreamExtended;
-import software.amazon.kinesis.leases.HierarchicalShardSyncer;
-import software.amazon.kinesis.leases.KinesisShardDetector;
-import software.amazon.kinesis.leases.LeaseCoordinator;
-import software.amazon.kinesis.leases.LeaseManagementFactory;
-import software.amazon.kinesis.leases.ShardDetector;
-import software.amazon.kinesis.leases.ShardSyncTaskManager;
-import software.amazon.kinesis.metrics.MetricsFactory;
-
-/**
- *
- */
-@Data
-@KinesisClientInternalApi
-public class DynamoDBLeaseManagementFactory implements LeaseManagementFactory {
- @NonNull
- private final KinesisAsyncClient kinesisClient;
- @NonNull
- private final String streamName;
- @NonNull
- private final DynamoDbAsyncClient dynamoDBClient;
- @NonNull
- private final String tableName;
- @NonNull
- private final String workerIdentifier;
- @NonNull
- private final ExecutorService executorService;
- @NonNull
- private final InitialPositionInStreamExtended initialPositionInStream;
- @NonNull
- private final HierarchicalShardSyncer hierarchicalShardSyncer;
-
- private final long failoverTimeMillis;
- private final long epsilonMillis;
- private final int maxLeasesForWorker;
- private final int maxLeasesToStealAtOneTime;
- private final int maxLeaseRenewalThreads;
- private final boolean cleanupLeasesUponShardCompletion;
- private final boolean ignoreUnexpectedChildShards;
- private final long shardSyncIntervalMillis;
- private final boolean consistentReads;
- private final long listShardsBackoffTimeMillis;
- private final int maxListShardsRetryAttempts;
- private final int maxCacheMissesBeforeReload;
- private final long listShardsCacheAllowedAgeInSeconds;
- private final int cacheMissWarningModulus;
- private final long initialLeaseTableReadCapacity;
- private final long initialLeaseTableWriteCapacity;
- private final TableCreatorCallback tableCreatorCallback;
-
- /**
- * Constructor.
- *
- * NOTE: This constructor is deprecated and will be removed in a future release.
- *
- * @param kinesisClient
- * @param streamName
- * @param dynamoDBClient
- * @param tableName
- * @param workerIdentifier
- * @param executorService
- * @param initialPositionInStream
- * @param failoverTimeMillis
- * @param epsilonMillis
- * @param maxLeasesForWorker
- * @param maxLeasesToStealAtOneTime
- * @param maxLeaseRenewalThreads
- * @param cleanupLeasesUponShardCompletion
- * @param ignoreUnexpectedChildShards
- * @param shardSyncIntervalMillis
- * @param consistentReads
- * @param listShardsBackoffTimeMillis
- * @param maxListShardsRetryAttempts
- * @param maxCacheMissesBeforeReload
- * @param listShardsCacheAllowedAgeInSeconds
- * @param cacheMissWarningModulus
- */
- @Deprecated
- public DynamoDBLeaseManagementFactory(final KinesisAsyncClient kinesisClient, final String streamName,
- final DynamoDbAsyncClient dynamoDBClient, final String tableName, final String workerIdentifier,
- final ExecutorService executorService, final InitialPositionInStreamExtended initialPositionInStream,
- final long failoverTimeMillis, final long epsilonMillis, final int maxLeasesForWorker,
- final int maxLeasesToStealAtOneTime, final int maxLeaseRenewalThreads,
- final boolean cleanupLeasesUponShardCompletion, final boolean ignoreUnexpectedChildShards,
- final long shardSyncIntervalMillis, final boolean consistentReads, final long listShardsBackoffTimeMillis,
- final int maxListShardsRetryAttempts, final int maxCacheMissesBeforeReload,
- final long listShardsCacheAllowedAgeInSeconds, final int cacheMissWarningModulus) {
- this(kinesisClient, streamName, dynamoDBClient, tableName, workerIdentifier, executorService,
- initialPositionInStream, failoverTimeMillis, epsilonMillis, maxLeasesForWorker,
- maxLeasesToStealAtOneTime, maxLeaseRenewalThreads, cleanupLeasesUponShardCompletion,
- ignoreUnexpectedChildShards, shardSyncIntervalMillis, consistentReads, listShardsBackoffTimeMillis,
- maxListShardsRetryAttempts, maxCacheMissesBeforeReload, listShardsCacheAllowedAgeInSeconds,
- cacheMissWarningModulus, TableConstants.DEFAULT_INITIAL_LEASE_TABLE_READ_CAPACITY,
- TableConstants.DEFAULT_INITIAL_LEASE_TABLE_WRITE_CAPACITY);
- }
-
- /**
- * Constructor.
- *
- *
- * NOTE: This constructor is deprecated and will be removed in a future release.
- *
- *
- * @param kinesisClient
- * @param streamName
- * @param dynamoDBClient
- * @param tableName
- * @param workerIdentifier
- * @param executorService
- * @param initialPositionInStream
- * @param failoverTimeMillis
- * @param epsilonMillis
- * @param maxLeasesForWorker
- * @param maxLeasesToStealAtOneTime
- * @param maxLeaseRenewalThreads
- * @param cleanupLeasesUponShardCompletion
- * @param ignoreUnexpectedChildShards
- * @param shardSyncIntervalMillis
- * @param consistentReads
- * @param listShardsBackoffTimeMillis
- * @param maxListShardsRetryAttempts
- * @param maxCacheMissesBeforeReload
- * @param listShardsCacheAllowedAgeInSeconds
- * @param cacheMissWarningModulus
- * @param initialLeaseTableReadCapacity
- * @param initialLeaseTableWriteCapacity
- */
- @Deprecated
- public DynamoDBLeaseManagementFactory(final KinesisAsyncClient kinesisClient, final String streamName,
- final DynamoDbAsyncClient dynamoDBClient, final String tableName, final String workerIdentifier,
- final ExecutorService executorService, final InitialPositionInStreamExtended initialPositionInStream,
- final long failoverTimeMillis, final long epsilonMillis, final int maxLeasesForWorker,
- final int maxLeasesToStealAtOneTime, final int maxLeaseRenewalThreads,
- final boolean cleanupLeasesUponShardCompletion, final boolean ignoreUnexpectedChildShards,
- final long shardSyncIntervalMillis, final boolean consistentReads, final long listShardsBackoffTimeMillis,
- final int maxListShardsRetryAttempts, final int maxCacheMissesBeforeReload,
- final long listShardsCacheAllowedAgeInSeconds, final int cacheMissWarningModulus,
- final long initialLeaseTableReadCapacity, final long initialLeaseTableWriteCapacity) {
- this(kinesisClient, streamName, dynamoDBClient, tableName, workerIdentifier, executorService,
- initialPositionInStream, failoverTimeMillis, epsilonMillis, maxLeasesForWorker,
- maxLeasesToStealAtOneTime, maxLeaseRenewalThreads, cleanupLeasesUponShardCompletion,
- ignoreUnexpectedChildShards, shardSyncIntervalMillis, consistentReads, listShardsBackoffTimeMillis,
- maxListShardsRetryAttempts, maxCacheMissesBeforeReload, listShardsCacheAllowedAgeInSeconds,
- cacheMissWarningModulus, initialLeaseTableReadCapacity, initialLeaseTableWriteCapacity,
- new HierarchicalShardSyncer(), TableCreatorCallback.NOOP_TABLE_CREATOR_CALLBACK);
- }
-
- /**
- * Constructor.
- *
- * @param kinesisClient
- * @param streamName
- * @param dynamoDBClient
- * @param tableName
- * @param workerIdentifier
- * @param executorService
- * @param initialPositionInStream
- * @param failoverTimeMillis
- * @param epsilonMillis
- * @param maxLeasesForWorker
- * @param maxLeasesToStealAtOneTime
- * @param maxLeaseRenewalThreads
- * @param cleanupLeasesUponShardCompletion
- * @param ignoreUnexpectedChildShards
- * @param shardSyncIntervalMillis
- * @param consistentReads
- * @param listShardsBackoffTimeMillis
- * @param maxListShardsRetryAttempts
- * @param maxCacheMissesBeforeReload
- * @param listShardsCacheAllowedAgeInSeconds
- * @param cacheMissWarningModulus
- * @param initialLeaseTableReadCapacity
- * @param initialLeaseTableWriteCapacity
- * @param hierarchicalShardSyncer
- * @param tableCreatorCallback
- */
- public DynamoDBLeaseManagementFactory(final KinesisAsyncClient kinesisClient, final String streamName,
- final DynamoDbAsyncClient dynamoDBClient, final String tableName, final String workerIdentifier,
- final ExecutorService executorService, final InitialPositionInStreamExtended initialPositionInStream,
- final long failoverTimeMillis, final long epsilonMillis, final int maxLeasesForWorker,
- final int maxLeasesToStealAtOneTime, final int maxLeaseRenewalThreads,
- final boolean cleanupLeasesUponShardCompletion, final boolean ignoreUnexpectedChildShards,
- final long shardSyncIntervalMillis, final boolean consistentReads, final long listShardsBackoffTimeMillis,
- final int maxListShardsRetryAttempts, final int maxCacheMissesBeforeReload,
- final long listShardsCacheAllowedAgeInSeconds, final int cacheMissWarningModulus,
- final long initialLeaseTableReadCapacity, final long initialLeaseTableWriteCapacity,
- final HierarchicalShardSyncer hierarchicalShardSyncer,
- final TableCreatorCallback tableCreatorCallback) {
- this.kinesisClient = kinesisClient;
- this.streamName = streamName;
- this.dynamoDBClient = dynamoDBClient;
- this.tableName = tableName;
- this.workerIdentifier = workerIdentifier;
- this.executorService = executorService;
- this.initialPositionInStream = initialPositionInStream;
- this.failoverTimeMillis = failoverTimeMillis;
- this.epsilonMillis = epsilonMillis;
- this.maxLeasesForWorker = maxLeasesForWorker;
- this.maxLeasesToStealAtOneTime = maxLeasesToStealAtOneTime;
- this.maxLeaseRenewalThreads = maxLeaseRenewalThreads;
- this.cleanupLeasesUponShardCompletion = cleanupLeasesUponShardCompletion;
- this.ignoreUnexpectedChildShards = ignoreUnexpectedChildShards;
- this.shardSyncIntervalMillis = shardSyncIntervalMillis;
- this.consistentReads = consistentReads;
- this.listShardsBackoffTimeMillis = listShardsBackoffTimeMillis;
- this.maxListShardsRetryAttempts = maxListShardsRetryAttempts;
- this.maxCacheMissesBeforeReload = maxCacheMissesBeforeReload;
- this.listShardsCacheAllowedAgeInSeconds = listShardsCacheAllowedAgeInSeconds;
- this.cacheMissWarningModulus = cacheMissWarningModulus;
- this.initialLeaseTableReadCapacity = initialLeaseTableReadCapacity;
- this.initialLeaseTableWriteCapacity = initialLeaseTableWriteCapacity;
- this.hierarchicalShardSyncer = hierarchicalShardSyncer;
- this.tableCreatorCallback = tableCreatorCallback;
- }
-
- @Override
- public LeaseCoordinator createLeaseCoordinator(@NonNull final MetricsFactory metricsFactory) {
- return new DynamoDBLeaseCoordinator(this.createLeaseRefresher(),
- workerIdentifier,
- failoverTimeMillis,
- epsilonMillis,
- maxLeasesForWorker,
- maxLeasesToStealAtOneTime,
- maxLeaseRenewalThreads,
- initialLeaseTableReadCapacity,
- initialLeaseTableWriteCapacity,
- metricsFactory);
- }
-
- @Override
- public ShardSyncTaskManager createShardSyncTaskManager(@NonNull final MetricsFactory metricsFactory) {
- return new ShardSyncTaskManager(this.createShardDetector(),
- this.createLeaseRefresher(),
- initialPositionInStream,
- cleanupLeasesUponShardCompletion,
- ignoreUnexpectedChildShards,
- shardSyncIntervalMillis,
- executorService,
- hierarchicalShardSyncer,
- metricsFactory);
- }
-
- @Override
- public DynamoDBLeaseRefresher createLeaseRefresher() {
- return new DynamoDBLeaseRefresher(tableName, dynamoDBClient, new DynamoDBLeaseSerializer(), consistentReads,
- tableCreatorCallback);
- }
-
- @Override
- public ShardDetector createShardDetector() {
- return new KinesisShardDetector(kinesisClient, streamName, listShardsBackoffTimeMillis,
- maxListShardsRetryAttempts, listShardsCacheAllowedAgeInSeconds, maxCacheMissesBeforeReload,
- cacheMissWarningModulus);
- }
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseRefresher.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseRefresher.java
deleted file mode 100644
index 79a12fc3..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseRefresher.java
+++ /dev/null
@@ -1,646 +0,0 @@
-/*
- * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-package software.amazon.kinesis.leases.dynamodb;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-
-import lombok.NonNull;
-import lombok.extern.slf4j.Slf4j;
-import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
-import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
-import software.amazon.awssdk.services.dynamodb.model.AttributeValueUpdate;
-import software.amazon.awssdk.services.dynamodb.model.ConditionalCheckFailedException;
-import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest;
-import software.amazon.awssdk.services.dynamodb.model.DeleteItemRequest;
-import software.amazon.awssdk.services.dynamodb.model.DescribeTableRequest;
-import software.amazon.awssdk.services.dynamodb.model.DescribeTableResponse;
-import software.amazon.awssdk.services.dynamodb.model.DynamoDbException;
-import software.amazon.awssdk.services.dynamodb.model.GetItemRequest;
-import software.amazon.awssdk.services.dynamodb.model.GetItemResponse;
-import software.amazon.awssdk.services.dynamodb.model.LimitExceededException;
-import software.amazon.awssdk.services.dynamodb.model.ProvisionedThroughput;
-import software.amazon.awssdk.services.dynamodb.model.ProvisionedThroughputExceededException;
-import software.amazon.awssdk.services.dynamodb.model.PutItemRequest;
-import software.amazon.awssdk.services.dynamodb.model.ResourceInUseException;
-import software.amazon.awssdk.services.dynamodb.model.ResourceNotFoundException;
-import software.amazon.awssdk.services.dynamodb.model.ScanRequest;
-import software.amazon.awssdk.services.dynamodb.model.ScanResponse;
-import software.amazon.awssdk.services.dynamodb.model.TableStatus;
-import software.amazon.awssdk.services.dynamodb.model.UpdateItemRequest;
-import software.amazon.awssdk.utils.CollectionUtils;
-import software.amazon.kinesis.annotations.KinesisClientInternalApi;
-import software.amazon.kinesis.leases.Lease;
-import software.amazon.kinesis.leases.LeaseRefresher;
-import software.amazon.kinesis.leases.LeaseSerializer;
-import software.amazon.kinesis.leases.exceptions.DependencyException;
-import software.amazon.kinesis.leases.exceptions.InvalidStateException;
-import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException;
-import software.amazon.kinesis.retrieval.AWSExceptionManager;
-import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber;
-
-/**
- * An implementation of {@link LeaseRefresher} that uses DynamoDB.
- */
-@Slf4j
-@KinesisClientInternalApi
-public class DynamoDBLeaseRefresher implements LeaseRefresher {
- protected final String table;
- protected final DynamoDbAsyncClient dynamoDBClient;
- protected final LeaseSerializer serializer;
- protected final boolean consistentReads;
- private final TableCreatorCallback tableCreatorCallback;
-
- private boolean newTableCreated = false;
-
- /**
- * Constructor.
- *
- *
- * NOTE: This constructor is deprecated and will be removed in a future release.
- *
- *
- * @param table
- * @param dynamoDBClient
- * @param serializer
- * @param consistentReads
- */
- @Deprecated
- public DynamoDBLeaseRefresher(final String table, final DynamoDbAsyncClient dynamoDBClient,
- final LeaseSerializer serializer, final boolean consistentReads) {
- this(table, dynamoDBClient, serializer, consistentReads, TableCreatorCallback.NOOP_TABLE_CREATOR_CALLBACK);
- }
-
- /**
- * Constructor.
- *
- * @param table
- * @param dynamoDBClient
- * @param serializer
- * @param consistentReads
- * @param tableCreatorCallback
- */
- public DynamoDBLeaseRefresher(final String table, final DynamoDbAsyncClient dynamoDBClient,
- final LeaseSerializer serializer, final boolean consistentReads,
- @NonNull final TableCreatorCallback tableCreatorCallback) {
- this.table = table;
- this.dynamoDBClient = dynamoDBClient;
- this.serializer = serializer;
- this.consistentReads = consistentReads;
- this.tableCreatorCallback = tableCreatorCallback;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public boolean createLeaseTableIfNotExists(@NonNull final Long readCapacity, @NonNull final Long writeCapacity)
- throws ProvisionedThroughputException, DependencyException {
- try {
- if (tableStatus() != null) {
- return newTableCreated;
- }
- } catch (DependencyException de) {
- //
- // Something went wrong with DynamoDB
- //
- log.error("Failed to get table status for {}", table, de);
- }
- ProvisionedThroughput throughput = ProvisionedThroughput.builder().readCapacityUnits(readCapacity)
- .writeCapacityUnits(writeCapacity).build();
- CreateTableRequest request = CreateTableRequest.builder().tableName(table).keySchema(serializer.getKeySchema())
- .attributeDefinitions(serializer.getAttributeDefinitions()).provisionedThroughput(throughput).build();
-
- final AWSExceptionManager exceptionManager = createExceptionManager();
- exceptionManager.add(ResourceInUseException.class, t -> t);
- exceptionManager.add(LimitExceededException.class, t -> t);
-
- try {
- try {
- dynamoDBClient.createTable(request).get();
- newTableCreated = true;
- } catch (ExecutionException e) {
- throw exceptionManager.apply(e.getCause());
- } catch (InterruptedException e) {
- throw new DependencyException(e);
- }
- } catch (ResourceInUseException e) {
- log.info("Table {} already exists.", table);
- return newTableCreated;
- } catch (LimitExceededException e) {
- throw new ProvisionedThroughputException("Capacity exceeded when creating table " + table, e);
- } catch (DynamoDbException e) {
- throw new DependencyException(e);
- }
- return newTableCreated;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public boolean leaseTableExists() throws DependencyException {
- return TableStatus.ACTIVE == tableStatus();
- }
-
- private TableStatus tableStatus() throws DependencyException {
- DescribeTableRequest request = DescribeTableRequest.builder().tableName(table).build();
-
- final AWSExceptionManager exceptionManager = createExceptionManager();
- exceptionManager.add(ResourceNotFoundException.class, t -> t);
-
- DescribeTableResponse result;
- try {
- try {
- result = dynamoDBClient.describeTable(request).get();
- } catch (ExecutionException e) {
- throw exceptionManager.apply(e.getCause());
- } catch (InterruptedException e) {
- // TODO: Check if this is the correct behavior
- throw new DependencyException(e);
- }
- } catch (ResourceNotFoundException e) {
- log.debug("Got ResourceNotFoundException for table {} in leaseTableExists, returning false.", table);
- return null;
- } catch (DynamoDbException e) {
- throw new DependencyException(e);
- }
-
- TableStatus tableStatus = result.table().tableStatus();
- log.debug("Lease table exists and is in status {}", tableStatus);
-
- return tableStatus;
- }
-
- @Override
- public boolean waitUntilLeaseTableExists(long secondsBetweenPolls, long timeoutSeconds) throws DependencyException {
- long sleepTimeRemaining = TimeUnit.SECONDS.toMillis(timeoutSeconds);
-
- while (!leaseTableExists()) {
- if (sleepTimeRemaining <= 0) {
- return false;
- }
-
- long timeToSleepMillis = Math.min(TimeUnit.SECONDS.toMillis(secondsBetweenPolls), sleepTimeRemaining);
-
- sleepTimeRemaining -= sleep(timeToSleepMillis);
- }
-
- if (newTableCreated) {
- log.debug("Lease table was recently created, will perform post table creation actions");
- performPostTableCreationAction();
- }
-
- return true;
- }
-
- /**
- * Exposed for testing purposes.
- *
- * @param timeToSleepMillis time to sleep in milliseconds
- *
- * @return actual time slept in millis
- */
- long sleep(long timeToSleepMillis) {
- long startTime = System.currentTimeMillis();
-
- try {
- Thread.sleep(timeToSleepMillis);
- } catch (InterruptedException e) {
- log.debug("Interrupted while sleeping");
- }
-
- return System.currentTimeMillis() - startTime;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public List listLeases() throws DependencyException, InvalidStateException, ProvisionedThroughputException {
- return list(null);
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public boolean isLeaseTableEmpty()
- throws DependencyException, InvalidStateException, ProvisionedThroughputException {
- return list(1).isEmpty();
- }
-
- /**
- * List with the given page size. Package access for integration testing.
- *
- * @param limit number of items to consider at a time - used by integration tests to force paging.
- * @return list of leases
- * @throws InvalidStateException if table does not exist
- * @throws DependencyException if DynamoDB scan fail in an unexpected way
- * @throws ProvisionedThroughputException if DynamoDB scan fail due to exceeded capacity
- */
- List list(Integer limit) throws DependencyException, InvalidStateException, ProvisionedThroughputException {
- log.debug("Listing leases from table {}", table);
-
- ScanRequest.Builder scanRequestBuilder = ScanRequest.builder().tableName(table);
- if (limit != null) {
- scanRequestBuilder = scanRequestBuilder.limit(limit);
- }
- ScanRequest scanRequest = scanRequestBuilder.build();
-
- final AWSExceptionManager exceptionManager = createExceptionManager();
- exceptionManager.add(ResourceNotFoundException.class, t -> t);
- exceptionManager.add(ProvisionedThroughputExceededException.class, t -> t);
-
- try {
- try {
- ScanResponse scanResult = dynamoDBClient.scan(scanRequest).get();
- List result = new ArrayList<>();
-
- while (scanResult != null) {
- for (Map item : scanResult.items()) {
- log.debug("Got item {} from DynamoDB.", item.toString());
- result.add(serializer.fromDynamoRecord(item));
- }
-
- Map lastEvaluatedKey = scanResult.lastEvaluatedKey();
- if (CollectionUtils.isNullOrEmpty(lastEvaluatedKey)) {
- // Signify that we're done.
- scanResult = null;
- log.debug("lastEvaluatedKey was null - scan finished.");
- } else {
- // Make another request, picking up where we left off.
- scanRequest = scanRequest.toBuilder().exclusiveStartKey(lastEvaluatedKey).build();
- log.debug("lastEvaluatedKey was {}, continuing scan.", lastEvaluatedKey);
- scanResult = dynamoDBClient.scan(scanRequest).get();
- }
- }
- log.debug("Listed {} leases from table {}", result.size(), table);
- return result;
- } catch (ExecutionException e) {
- throw exceptionManager.apply(e.getCause());
- } catch (InterruptedException e) {
- // TODO: Check if this is the correct behavior
- throw new DependencyException(e);
- }
- } catch (ResourceNotFoundException e) {
- throw new InvalidStateException("Cannot scan lease table " + table + " because it does not exist.", e);
- } catch (ProvisionedThroughputExceededException e) {
- throw new ProvisionedThroughputException(e);
- } catch (DynamoDbException e) {
- throw new DependencyException(e);
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public boolean createLeaseIfNotExists(@NonNull final Lease lease)
- throws DependencyException, InvalidStateException, ProvisionedThroughputException {
- log.debug("Creating lease {}", lease);
-
- PutItemRequest request = PutItemRequest.builder().tableName(table).item(serializer.toDynamoRecord(lease))
- .expected(serializer.getDynamoNonexistantExpectation()).build();
-
- final AWSExceptionManager exceptionManager = createExceptionManager();
- exceptionManager.add(ConditionalCheckFailedException.class, t -> t);
-
- try {
- try {
- dynamoDBClient.putItem(request).get();
- } catch (ExecutionException e) {
- throw exceptionManager.apply(e.getCause());
- } catch (InterruptedException e) {
- // TODO: Check if this is the correct behavior
- throw new DependencyException(e);
- }
- } catch (ConditionalCheckFailedException e) {
- log.debug("Did not create lease {} because it already existed", lease);
- return false;
- } catch (DynamoDbException e) {
- throw convertAndRethrowExceptions("create", lease.leaseKey(), e);
- }
- return true;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public Lease getLease(@NonNull final String leaseKey)
- throws DependencyException, InvalidStateException, ProvisionedThroughputException {
- log.debug("Getting lease with key {}", leaseKey);
-
- GetItemRequest request = GetItemRequest.builder().tableName(table).key(serializer.getDynamoHashKey(leaseKey))
- .consistentRead(consistentReads).build();
- final AWSExceptionManager exceptionManager = createExceptionManager();
- try {
- try {
- GetItemResponse result = dynamoDBClient.getItem(request).get();
-
- Map dynamoRecord = result.item();
- if (CollectionUtils.isNullOrEmpty(dynamoRecord)) {
- log.debug("No lease found with key {}, returning null.", leaseKey);
- return null;
- } else {
- final Lease lease = serializer.fromDynamoRecord(dynamoRecord);
- log.debug("Got lease {}", lease);
- return lease;
- }
- } catch (ExecutionException e) {
- throw exceptionManager.apply(e.getCause());
- } catch (InterruptedException e) {
- // TODO: check behavior
- throw new DependencyException(e);
- }
- } catch (DynamoDbException e) {
- throw convertAndRethrowExceptions("get", leaseKey, e);
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public boolean renewLease(@NonNull final Lease lease)
- throws DependencyException, InvalidStateException, ProvisionedThroughputException {
- log.debug("Renewing lease with key {}", lease.leaseKey());
-
- UpdateItemRequest request = UpdateItemRequest.builder().tableName(table).key(serializer.getDynamoHashKey(lease))
- .expected(serializer.getDynamoLeaseCounterExpectation(lease))
- .attributeUpdates(serializer.getDynamoLeaseCounterUpdate(lease)).build();
-
- final AWSExceptionManager exceptionManager = createExceptionManager();
- exceptionManager.add(ConditionalCheckFailedException.class, t -> t);
-
- try {
- try {
- dynamoDBClient.updateItem(request).get();
- } catch (ExecutionException e) {
- throw exceptionManager.apply(e.getCause());
- } catch (InterruptedException e) {
- // TODO: Check if this is correct behavior
- throw new DependencyException(e);
- }
- } catch (ConditionalCheckFailedException e) {
- log.debug("Lease renewal failed for lease with key {} because the lease counter was not {}",
- lease.leaseKey(), lease.leaseCounter());
-
- // If we had a spurious retry during the Dynamo update, then this conditional PUT failure
- // might be incorrect. So, we get the item straight away and check if the lease owner + lease
- // counter are what we expected.
- String expectedOwner = lease.leaseOwner();
- Long expectedCounter = lease.leaseCounter() + 1;
- final Lease updatedLease = getLease(lease.leaseKey());
- if (updatedLease == null || !expectedOwner.equals(updatedLease.leaseOwner())
- || !expectedCounter.equals(updatedLease.leaseCounter())) {
- return false;
- }
-
- log.info("Detected spurious renewal failure for lease with key {}, but recovered", lease.leaseKey());
- } catch (DynamoDbException e) {
- throw new DependencyException(e);
- }
-
- lease.leaseCounter(lease.leaseCounter() + 1);
- return true;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public boolean takeLease(@NonNull final Lease lease, @NonNull final String owner)
- throws DependencyException, InvalidStateException, ProvisionedThroughputException {
- final String oldOwner = lease.leaseOwner();
-
- log.debug("Taking lease with leaseKey {} from {} to {}", lease.leaseKey(),
- lease.leaseOwner() == null ? "nobody" : lease.leaseOwner(), owner);
-
- final AWSExceptionManager exceptionManager = createExceptionManager();
- exceptionManager.add(ConditionalCheckFailedException.class, t -> t);
-
- Map updates = serializer.getDynamoLeaseCounterUpdate(lease);
- updates.putAll(serializer.getDynamoTakeLeaseUpdate(lease, owner));
-
- UpdateItemRequest request = UpdateItemRequest.builder().tableName(table).key(serializer.getDynamoHashKey(lease))
- .expected(serializer.getDynamoLeaseCounterExpectation(lease)).attributeUpdates(updates).build();
-
- try {
- try {
- dynamoDBClient.updateItem(request).get();
- } catch (ExecutionException e) {
- throw exceptionManager.apply(e.getCause());
- } catch (InterruptedException e) {
- // TODO: Check behavior
- throw new DependencyException(e);
- }
- } catch (ConditionalCheckFailedException e) {
- log.debug("Lease renewal failed for lease with key {} because the lease counter was not {}",
- lease.leaseKey(), lease.leaseCounter());
- return false;
- } catch (DynamoDbException e) {
- throw convertAndRethrowExceptions("take", lease.leaseKey(), e);
- }
-
- lease.leaseCounter(lease.leaseCounter() + 1);
- lease.leaseOwner(owner);
-
- if (oldOwner != null && !oldOwner.equals(owner)) {
- lease.ownerSwitchesSinceCheckpoint(lease.ownerSwitchesSinceCheckpoint() + 1);
- }
-
- return true;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public boolean evictLease(@NonNull final Lease lease)
- throws DependencyException, InvalidStateException, ProvisionedThroughputException {
- log.debug("Evicting lease with leaseKey {} owned by {}", lease.leaseKey(), lease.leaseOwner());
-
- final AWSExceptionManager exceptionManager = createExceptionManager();
- exceptionManager.add(ConditionalCheckFailedException.class, t -> t);
-
- Map updates = serializer.getDynamoLeaseCounterUpdate(lease);
- updates.putAll(serializer.getDynamoEvictLeaseUpdate(lease));
- UpdateItemRequest request = UpdateItemRequest.builder().tableName(table).key(serializer.getDynamoHashKey(lease))
- .expected(serializer.getDynamoLeaseOwnerExpectation(lease)).attributeUpdates(updates).build();
-
- try {
- try {
- dynamoDBClient.updateItem(request).get();
- } catch (ExecutionException e) {
- throw exceptionManager.apply(e.getCause());
- } catch (InterruptedException e) {
- // TODO: check behavior
- throw new DependencyException(e);
- }
- } catch (ConditionalCheckFailedException e) {
- log.debug("Lease eviction failed for lease with key {} because the lease owner was not {}",
- lease.leaseKey(), lease.leaseOwner());
- return false;
- } catch (DynamoDbException e) {
- throw convertAndRethrowExceptions("evict", lease.leaseKey(), e);
- }
-
- lease.leaseOwner(null);
- lease.leaseCounter(lease.leaseCounter() + 1);
- return true;
- }
-
- /**
- * {@inheritDoc}
- */
- public void deleteAll() throws DependencyException, InvalidStateException, ProvisionedThroughputException {
- List allLeases = listLeases();
-
- log.warn("Deleting {} items from table {}", allLeases.size(), table);
-
- final AWSExceptionManager exceptionManager = createExceptionManager();
- for (final Lease lease : allLeases) {
- DeleteItemRequest deleteRequest = DeleteItemRequest.builder().tableName(table)
- .key(serializer.getDynamoHashKey(lease)).build();
-
- try {
- try {
- dynamoDBClient.deleteItem(deleteRequest).get();
- } catch (ExecutionException e) {
- throw exceptionManager.apply(e.getCause());
- } catch (InterruptedException e) {
- // TODO: check the behavior
- throw new DependencyException(e);
- }
- } catch (DynamoDbException e) {
- throw convertAndRethrowExceptions("deleteAll", lease.leaseKey(), e);
- }
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void deleteLease(@NonNull final Lease lease)
- throws DependencyException, InvalidStateException, ProvisionedThroughputException {
- log.debug("Deleting lease with leaseKey {}", lease.leaseKey());
-
- DeleteItemRequest deleteRequest = DeleteItemRequest.builder().tableName(table)
- .key(serializer.getDynamoHashKey(lease)).build();
-
- final AWSExceptionManager exceptionManager = createExceptionManager();
- try {
- try {
- dynamoDBClient.deleteItem(deleteRequest).get();
- } catch (ExecutionException e) {
- throw exceptionManager.apply(e.getCause());
- } catch (InterruptedException e) {
- // TODO: Check if this is the correct behavior
- throw new DependencyException(e);
- }
- } catch (DynamoDbException e) {
- throw convertAndRethrowExceptions("delete", lease.leaseKey(), e);
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public boolean updateLease(@NonNull final Lease lease)
- throws DependencyException, InvalidStateException, ProvisionedThroughputException {
- log.debug("Updating lease {}", lease);
-
- final AWSExceptionManager exceptionManager = createExceptionManager();
- exceptionManager.add(ConditionalCheckFailedException.class, t -> t);
-
- Map updates = serializer.getDynamoLeaseCounterUpdate(lease);
- updates.putAll(serializer.getDynamoUpdateLeaseUpdate(lease));
-
- UpdateItemRequest request = UpdateItemRequest.builder().tableName(table).key(serializer.getDynamoHashKey(lease))
- .expected(serializer.getDynamoLeaseCounterExpectation(lease)).attributeUpdates(updates).build();
-
- try {
- try {
- dynamoDBClient.updateItem(request).get();
- } catch (ExecutionException e) {
- throw exceptionManager.apply(e.getCause());
- } catch (InterruptedException e) {
- throw new DependencyException(e);
- }
- } catch (ConditionalCheckFailedException e) {
- log.debug("Lease update failed for lease with key {} because the lease counter was not {}",
- lease.leaseKey(), lease.leaseCounter());
- return false;
- } catch (DynamoDbException e) {
- throw convertAndRethrowExceptions("update", lease.leaseKey(), e);
- }
-
- lease.leaseCounter(lease.leaseCounter() + 1);
- return true;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public ExtendedSequenceNumber getCheckpoint(String shardId)
- throws ProvisionedThroughputException, InvalidStateException, DependencyException {
- ExtendedSequenceNumber checkpoint = null;
- Lease lease = getLease(shardId);
- if (lease != null) {
- checkpoint = lease.checkpoint();
- }
- return checkpoint;
- }
-
- /*
- * This method contains boilerplate exception handling - it throws or returns something to be thrown. The
- * inconsistency there exists to satisfy the compiler when this method is used at the end of non-void methods.
- */
- protected DependencyException convertAndRethrowExceptions(String operation, String leaseKey, Exception e)
- throws ProvisionedThroughputException, InvalidStateException {
- if (e instanceof ProvisionedThroughputExceededException) {
- log.warn("Provisioned Throughput on the lease table has been exceeded. It's recommended that you increase"
- + " the IOPs on the table. Failure to increase the IOPs may cause the application to not make"
- + " progress.");
- throw new ProvisionedThroughputException(e);
- } else if (e instanceof ResourceNotFoundException) {
- throw new InvalidStateException(
- String.format("Cannot %s lease with key %s because table %s does not exist.",
- operation, leaseKey, table),
- e);
- } else {
- return new DependencyException(e);
- }
- }
-
- private AWSExceptionManager createExceptionManager() {
- final AWSExceptionManager exceptionManager = new AWSExceptionManager();
- exceptionManager.add(DynamoDbException.class, t -> t);
- return exceptionManager;
- }
-
- void performPostTableCreationAction() {
- tableCreatorCallback.performAction(
- TableCreatorCallbackInput.builder().dynamoDbClient(dynamoDBClient).tableName(table).build());
- }
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseRenewer.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseRenewer.java
deleted file mode 100644
index 35251e99..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseRenewer.java
+++ /dev/null
@@ -1,421 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-package software.amazon.kinesis.leases.dynamodb;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentNavigableMap;
-import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.lang3.StringUtils;
-
-import lombok.NonNull;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import software.amazon.awssdk.services.cloudwatch.model.StandardUnit;
-import software.amazon.kinesis.annotations.KinesisClientInternalApi;
-import software.amazon.kinesis.leases.Lease;
-import software.amazon.kinesis.leases.LeaseRefresher;
-import software.amazon.kinesis.leases.LeaseRenewer;
-import software.amazon.kinesis.leases.exceptions.DependencyException;
-import software.amazon.kinesis.leases.exceptions.InvalidStateException;
-import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException;
-import software.amazon.kinesis.metrics.MetricsFactory;
-import software.amazon.kinesis.metrics.MetricsLevel;
-import software.amazon.kinesis.metrics.MetricsScope;
-import software.amazon.kinesis.metrics.MetricsUtil;
-
-/**
- * An implementation of {@link LeaseRenewer} that uses DynamoDB via {@link LeaseRefresher}.
- */
-@Slf4j
-@KinesisClientInternalApi
-public class DynamoDBLeaseRenewer implements LeaseRenewer {
- private static final int RENEWAL_RETRIES = 2;
- private static final String RENEW_ALL_LEASES_DIMENSION = "RenewAllLeases";
-
- private final LeaseRefresher leaseRefresher;
- private final String workerIdentifier;
- private final long leaseDurationNanos;
- private final ExecutorService executorService;
- private final MetricsFactory metricsFactory;
-
- private final ConcurrentNavigableMap ownedLeases = new ConcurrentSkipListMap<>();
-
- /**
- * Constructor.
- *
- * @param leaseRefresher
- * LeaseRefresher to use
- * @param workerIdentifier
- * identifier of this worker
- * @param leaseDurationMillis
- * duration of a lease in milliseconds
- * @param executorService
- * ExecutorService to use for renewing leases in parallel
- */
- public DynamoDBLeaseRenewer(final LeaseRefresher leaseRefresher, final String workerIdentifier,
- final long leaseDurationMillis, final ExecutorService executorService,
- final MetricsFactory metricsFactory) {
- this.leaseRefresher = leaseRefresher;
- this.workerIdentifier = workerIdentifier;
- this.leaseDurationNanos = TimeUnit.MILLISECONDS.toNanos(leaseDurationMillis);
- this.executorService = executorService;
- this.metricsFactory = metricsFactory;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void renewLeases() throws DependencyException, InvalidStateException {
- // Due to the eventually consistent nature of ConcurrentNavigableMap iterators, this log entry may become
- // inaccurate during iteration.
- log.debug("Worker {} holding {} leases: {}", workerIdentifier, ownedLeases.size(), ownedLeases);
-
- /*
- * Lease renewals are done in parallel so many leases can be renewed for short lease fail over time
- * configuration. In this case, metrics scope is also shared across different threads, so scope must be thread
- * safe.
- */
- final MetricsScope scope = MetricsUtil.createMetricsWithOperation(metricsFactory, RENEW_ALL_LEASES_DIMENSION);
-
- long startTime = System.currentTimeMillis();
- boolean success = false;
-
- try {
- /*
- * We iterate in descending order here so that the synchronized(lease) inside renewLease doesn't "lead" calls
- * to getCurrentlyHeldLeases. They'll still cross paths, but they won't interleave their executions.
- */
- int lostLeases = 0;
- List> renewLeaseTasks = new ArrayList<>();
- for (Lease lease : ownedLeases.descendingMap().values()) {
- renewLeaseTasks.add(executorService.submit(new RenewLeaseTask(lease)));
- }
- int leasesInUnknownState = 0;
- Exception lastException = null;
- for (Future renewLeaseTask : renewLeaseTasks) {
- try {
- if (!renewLeaseTask.get()) {
- lostLeases++;
- }
- } catch (InterruptedException e) {
- log.info("Interrupted while waiting for a lease to renew.");
- leasesInUnknownState += 1;
- Thread.currentThread().interrupt();
- } catch (ExecutionException e) {
- log.error("Encountered an exception while renewing a lease.", e.getCause());
- leasesInUnknownState += 1;
- lastException = e;
- }
- }
-
- scope.addData("LostLeases", lostLeases, StandardUnit.COUNT, MetricsLevel.SUMMARY);
- scope.addData("CurrentLeases", ownedLeases.size(), StandardUnit.COUNT, MetricsLevel.SUMMARY);
- if (leasesInUnknownState > 0) {
- throw new DependencyException(
- String.format("Encountered an exception while renewing leases. The number"
- + " of leases which might not have been renewed is %d", leasesInUnknownState),
- lastException);
- }
- success = true;
- } finally {
- MetricsUtil.addWorkerIdentifier(scope, workerIdentifier);
- MetricsUtil.addSuccessAndLatency(scope, success, startTime, MetricsLevel.SUMMARY);
- MetricsUtil.endScope(scope);
- }
- }
-
- @RequiredArgsConstructor
- private class RenewLeaseTask implements Callable {
- private final Lease lease;
-
- @Override
- public Boolean call() throws Exception {
- return renewLease(lease);
- }
- }
-
- private boolean renewLease(Lease lease) throws DependencyException, InvalidStateException {
- return renewLease(lease, false);
- }
-
- private boolean renewLease(Lease lease, boolean renewEvenIfExpired) throws DependencyException, InvalidStateException {
- String leaseKey = lease.leaseKey();
-
- final MetricsScope scope = MetricsUtil.createMetricsWithOperation(metricsFactory, RENEW_ALL_LEASES_DIMENSION);
-
- boolean success = false;
- boolean renewedLease = false;
- long startTime = System.currentTimeMillis();
- try {
- for (int i = 1; i <= RENEWAL_RETRIES; i++) {
- try {
- synchronized (lease) {
- // Don't renew expired lease during regular renewals. getCopyOfHeldLease may have returned null
- // triggering the application processing to treat this as a lost lease (fail checkpoint with
- // ShutdownException).
- boolean isLeaseExpired = lease.isExpired(leaseDurationNanos, System.nanoTime());
- if (renewEvenIfExpired || !isLeaseExpired) {
- renewedLease = leaseRefresher.renewLease(lease);
- }
- if (renewedLease) {
- lease.lastCounterIncrementNanos(System.nanoTime());
- }
- }
-
- if (renewedLease) {
- if (log.isDebugEnabled()) {
- log.debug("Worker {} successfully renewed lease with key {}", workerIdentifier, leaseKey);
- }
- } else {
- log.info("Worker {} lost lease with key {}", workerIdentifier, leaseKey);
- ownedLeases.remove(leaseKey);
- }
-
- success = true;
- break;
- } catch (ProvisionedThroughputException e) {
- log.info("Worker {} could not renew lease with key {} on try {} out of {} due to capacity",
- workerIdentifier, leaseKey, i, RENEWAL_RETRIES);
- }
- }
- } finally {
- MetricsUtil.addWorkerIdentifier(scope, workerIdentifier);
- MetricsUtil.addSuccessAndLatency(scope, "RenewLease", success, startTime, MetricsLevel.DETAILED);
- MetricsUtil.endScope(scope);
- }
-
- return renewedLease;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public Map getCurrentlyHeldLeases() {
- Map result = new HashMap<>();
- long now = System.nanoTime();
-
- for (String leaseKey : ownedLeases.keySet()) {
- Lease copy = getCopyOfHeldLease(leaseKey, now);
- if (copy != null) {
- result.put(copy.leaseKey(), copy);
- }
- }
-
- return result;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public Lease getCurrentlyHeldLease(String leaseKey) {
- return getCopyOfHeldLease(leaseKey, System.nanoTime());
- }
-
- /**
- * Internal method to return a lease with a specific lease key only if we currently hold it.
- *
- * @param leaseKey key of lease to return
- * @param now current timestamp for old-ness checking
- * @return non-authoritative copy of the held lease, or null if we don't currently hold it
- */
- private Lease getCopyOfHeldLease(String leaseKey, long now) {
- Lease authoritativeLease = ownedLeases.get(leaseKey);
- if (authoritativeLease == null) {
- return null;
- } else {
- Lease copy = null;
- synchronized (authoritativeLease) {
- copy = authoritativeLease.copy();
- }
-
- if (copy.isExpired(leaseDurationNanos, now)) {
- log.info("getCurrentlyHeldLease not returning lease with key {} because it is expired",
- copy.leaseKey());
- return null;
- } else {
- return copy;
- }
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public boolean updateLease(Lease lease, UUID concurrencyToken, @NonNull String operation, String shardId)
- throws DependencyException, InvalidStateException, ProvisionedThroughputException {
- verifyNotNull(lease, "lease cannot be null");
- verifyNotNull(lease.leaseKey(), "leaseKey cannot be null");
- verifyNotNull(concurrencyToken, "concurrencyToken cannot be null");
-
- String leaseKey = lease.leaseKey();
- Lease authoritativeLease = ownedLeases.get(leaseKey);
-
- if (authoritativeLease == null) {
- log.info("Worker {} could not update lease with key {} because it does not hold it", workerIdentifier,
- leaseKey);
- return false;
- }
-
- /*
- * If the passed-in concurrency token doesn't match the concurrency token of the authoritative lease, it means
- * the lease was lost and regained between when the caller acquired his concurrency token and when the caller
- * called update.
- */
- if (!authoritativeLease.concurrencyToken().equals(concurrencyToken)) {
- log.info("Worker {} refusing to update lease with key {} because concurrency tokens don't match",
- workerIdentifier, leaseKey);
- return false;
- }
-
- final MetricsScope scope = MetricsUtil.createMetricsWithOperation(metricsFactory, operation);
- if (StringUtils.isNotEmpty(shardId)) {
- MetricsUtil.addShardId(scope, shardId);
- }
-
- long startTime = System.currentTimeMillis();
- boolean success = false;
- try {
- synchronized (authoritativeLease) {
- authoritativeLease.update(lease);
- boolean updatedLease = leaseRefresher.updateLease(authoritativeLease);
- if (updatedLease) {
- // Updates increment the counter
- authoritativeLease.lastCounterIncrementNanos(System.nanoTime());
- } else {
- /*
- * If updateLease returns false, it means someone took the lease from us. Remove the lease
- * from our set of owned leases pro-actively rather than waiting for a run of renewLeases().
- */
- log.info("Worker {} lost lease with key {} - discovered during update", workerIdentifier, leaseKey);
-
- /*
- * Remove only if the value currently in the map is the same as the authoritative lease. We're
- * guarding against a pause after the concurrency token check above. It plays out like so:
- *
- * 1) Concurrency token check passes
- * 2) Pause. Lose lease, re-acquire lease. This requires at least one lease counter update.
- * 3) Unpause. leaseRefresher.updateLease fails conditional write due to counter updates, returns
- * false.
- * 4) ownedLeases.remove(key, value) doesn't do anything because authoritativeLease does not
- * .equals() the re-acquired version in the map on the basis of lease counter. This is what we want.
- * If we just used ownedLease.remove(key), we would have pro-actively removed a lease incorrectly.
- *
- * Note that there is a subtlety here - Lease.equals() deliberately does not check the concurrency
- * token, but it does check the lease counter, so this scheme works.
- */
- ownedLeases.remove(leaseKey, authoritativeLease);
- }
-
- success = true;
- return updatedLease;
- }
- } finally {
- MetricsUtil.addSuccessAndLatency(scope, "UpdateLease", success, startTime, MetricsLevel.DETAILED);
- MetricsUtil.endScope(scope);
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void addLeasesToRenew(Collection newLeases) {
- verifyNotNull(newLeases, "newLeases cannot be null");
-
- for (Lease lease : newLeases) {
- if (lease.lastCounterIncrementNanos() == null) {
- log.info("addLeasesToRenew ignoring lease with key {} because it does not have lastRenewalNanos set",
- lease.leaseKey());
- continue;
- }
-
- Lease authoritativeLease = lease.copy();
-
- /*
- * Assign a concurrency token when we add this to the set of currently owned leases. This ensures that
- * every time we acquire a lease, it gets a new concurrency token.
- */
- authoritativeLease.concurrencyToken(UUID.randomUUID());
- ownedLeases.put(authoritativeLease.leaseKey(), authoritativeLease);
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void clearCurrentlyHeldLeases() {
- ownedLeases.clear();
- }
-
- /**
- * {@inheritDoc}
- * @param lease the lease to drop.
- */
- @Override
- public void dropLease(Lease lease) {
- ownedLeases.remove(lease.leaseKey());
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void initialize() throws DependencyException, InvalidStateException, ProvisionedThroughputException {
- Collection leases = leaseRefresher.listLeases();
- List myLeases = new LinkedList<>();
- boolean renewEvenIfExpired = true;
-
- for (Lease lease : leases) {
- if (workerIdentifier.equals(lease.leaseOwner())) {
- log.info(" Worker {} found lease {}", workerIdentifier, lease);
- // Okay to renew even if lease is expired, because we start with an empty list and we add the lease to
- // our list only after a successful renew. So we don't need to worry about the edge case where we could
- // continue renewing a lease after signaling a lease loss to the application.
-
- if (renewLease(lease, renewEvenIfExpired)) {
- myLeases.add(lease);
- }
- } else {
- log.debug("Worker {} ignoring lease {} ", workerIdentifier, lease);
- }
- }
-
- addLeasesToRenew(myLeases);
- }
-
- private void verifyNotNull(Object object, String message) {
- if (object == null) {
- throw new IllegalArgumentException(message);
- }
- }
-
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseSerializer.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseSerializer.java
deleted file mode 100644
index 4b58a429..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseSerializer.java
+++ /dev/null
@@ -1,238 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-package software.amazon.kinesis.leases.dynamodb;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-
-import com.google.common.base.Strings;
-import software.amazon.awssdk.services.dynamodb.model.AttributeAction;
-import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
-import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
-import software.amazon.awssdk.services.dynamodb.model.AttributeValueUpdate;
-import software.amazon.awssdk.services.dynamodb.model.ExpectedAttributeValue;
-import software.amazon.awssdk.services.dynamodb.model.KeySchemaElement;
-import software.amazon.awssdk.services.dynamodb.model.KeyType;
-import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
-import software.amazon.kinesis.annotations.KinesisClientInternalApi;
-import software.amazon.kinesis.leases.DynamoUtils;
-import software.amazon.kinesis.leases.Lease;
-import software.amazon.kinesis.leases.LeaseSerializer;
-import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber;
-
-/**
- * An implementation of ILeaseSerializer for basic Lease objects. Can also instantiate subclasses of Lease so that
- * LeaseSerializer can be decorated by other classes if you need to add fields to leases.
- */
-@KinesisClientInternalApi
-public class DynamoDBLeaseSerializer implements LeaseSerializer {
- private static final String LEASE_KEY_KEY = "leaseKey";
- private static final String LEASE_OWNER_KEY = "leaseOwner";
- private static final String LEASE_COUNTER_KEY = "leaseCounter";
- private static final String OWNER_SWITCHES_KEY = "ownerSwitchesSinceCheckpoint";
- private static final String CHECKPOINT_SEQUENCE_NUMBER_KEY = "checkpoint";
- private static final String CHECKPOINT_SUBSEQUENCE_NUMBER_KEY = "checkpointSubSequenceNumber";
- private static final String PENDING_CHECKPOINT_SEQUENCE_KEY = "pendingCheckpoint";
- private static final String PENDING_CHECKPOINT_SUBSEQUENCE_KEY = "pendingCheckpointSubSequenceNumber";
- private static final String PARENT_SHARD_ID_KEY = "parentShardId";
-
- @Override
- public Map toDynamoRecord(final Lease lease) {
- Map result = new HashMap<>();
-
- result.put(LEASE_KEY_KEY, DynamoUtils.createAttributeValue(lease.leaseKey()));
- result.put(LEASE_COUNTER_KEY, DynamoUtils.createAttributeValue(lease.leaseCounter()));
-
- if (lease.leaseOwner() != null) {
- result.put(LEASE_OWNER_KEY, DynamoUtils.createAttributeValue(lease.leaseOwner()));
- }
-
- result.put(OWNER_SWITCHES_KEY, DynamoUtils.createAttributeValue(lease.ownerSwitchesSinceCheckpoint()));
- result.put(CHECKPOINT_SEQUENCE_NUMBER_KEY, DynamoUtils.createAttributeValue(lease.checkpoint().sequenceNumber()));
- result.put(CHECKPOINT_SUBSEQUENCE_NUMBER_KEY, DynamoUtils.createAttributeValue(lease.checkpoint().subSequenceNumber()));
- if (lease.parentShardIds() != null && !lease.parentShardIds().isEmpty()) {
- result.put(PARENT_SHARD_ID_KEY, DynamoUtils.createAttributeValue(lease.parentShardIds()));
- }
-
- if (lease.pendingCheckpoint() != null && !lease.pendingCheckpoint().sequenceNumber().isEmpty()) {
- result.put(PENDING_CHECKPOINT_SEQUENCE_KEY, DynamoUtils.createAttributeValue(lease.pendingCheckpoint().sequenceNumber()));
- result.put(PENDING_CHECKPOINT_SUBSEQUENCE_KEY, DynamoUtils.createAttributeValue(lease.pendingCheckpoint().subSequenceNumber()));
- }
-
- return result;
- }
-
- @Override
- public Lease fromDynamoRecord(final Map dynamoRecord) {
- Lease result = new Lease();
- result.leaseKey(DynamoUtils.safeGetString(dynamoRecord, LEASE_KEY_KEY));
- result.leaseOwner(DynamoUtils.safeGetString(dynamoRecord, LEASE_OWNER_KEY));
- result.leaseCounter(DynamoUtils.safeGetLong(dynamoRecord, LEASE_COUNTER_KEY));
-
- result.ownerSwitchesSinceCheckpoint(DynamoUtils.safeGetLong(dynamoRecord, OWNER_SWITCHES_KEY));
- result.checkpoint(
- new ExtendedSequenceNumber(
- DynamoUtils.safeGetString(dynamoRecord, CHECKPOINT_SEQUENCE_NUMBER_KEY),
- DynamoUtils.safeGetLong(dynamoRecord, CHECKPOINT_SUBSEQUENCE_NUMBER_KEY))
- );
- result.parentShardIds(DynamoUtils.safeGetSS(dynamoRecord, PARENT_SHARD_ID_KEY));
-
- if (!Strings.isNullOrEmpty(DynamoUtils.safeGetString(dynamoRecord, PENDING_CHECKPOINT_SEQUENCE_KEY))) {
- result.pendingCheckpoint(
- new ExtendedSequenceNumber(
- DynamoUtils.safeGetString(dynamoRecord, PENDING_CHECKPOINT_SEQUENCE_KEY),
- DynamoUtils.safeGetLong(dynamoRecord, PENDING_CHECKPOINT_SUBSEQUENCE_KEY))
- );
- }
-
- return result;
- }
-
- @Override
- public Map getDynamoHashKey(final String leaseKey) {
- Map result = new HashMap<>();
-
- result.put(LEASE_KEY_KEY, DynamoUtils.createAttributeValue(leaseKey));
-
- return result;
- }
-
- @Override
- public Map getDynamoHashKey(final Lease lease) {
- return getDynamoHashKey(lease.leaseKey());
- }
-
- @Override
- public Map getDynamoLeaseCounterExpectation(final Lease lease) {
- return getDynamoLeaseCounterExpectation(lease.leaseCounter());
- }
-
- public Map getDynamoLeaseCounterExpectation(final Long leaseCounter) {
- Map result = new HashMap<>();
-
- ExpectedAttributeValue eav = ExpectedAttributeValue.builder().value(DynamoUtils.createAttributeValue(leaseCounter)).build();
- result.put(LEASE_COUNTER_KEY, eav);
-
- return result;
- }
-
- @Override
- public Map getDynamoLeaseOwnerExpectation(final Lease lease) {
- Map result = new HashMap<>();
-
- ExpectedAttributeValue.Builder eavBuilder = ExpectedAttributeValue.builder();
-
- if (lease.leaseOwner() == null) {
- eavBuilder = eavBuilder.exists(false);
- } else {
- eavBuilder = eavBuilder.value(DynamoUtils.createAttributeValue(lease.leaseOwner()));
- }
-
- result.put(LEASE_OWNER_KEY, eavBuilder.build());
-
- return result;
- }
-
- @Override
- public Map getDynamoNonexistantExpectation() {
- Map result = new HashMap<>();
-
- ExpectedAttributeValue expectedAV = ExpectedAttributeValue.builder().exists(false).build();
- result.put(LEASE_KEY_KEY, expectedAV);
-
- return result;
- }
-
- @Override
- public Map getDynamoLeaseCounterUpdate(final Lease lease) {
- return getDynamoLeaseCounterUpdate(lease.leaseCounter());
- }
-
- public Map getDynamoLeaseCounterUpdate(Long leaseCounter) {
- Map result = new HashMap<>();
-
- AttributeValueUpdate avu =
- AttributeValueUpdate.builder().value(DynamoUtils.createAttributeValue(leaseCounter + 1)).action(AttributeAction.PUT).build();
- result.put(LEASE_COUNTER_KEY, avu);
-
- return result;
- }
-
- @Override
- public Map getDynamoTakeLeaseUpdate(final Lease lease, String owner) {
- Map result = new HashMap<>();
-
- result.put(LEASE_OWNER_KEY, AttributeValueUpdate.builder().value(DynamoUtils.createAttributeValue(owner)).action(AttributeAction.PUT).build());
-
- String oldOwner = lease.leaseOwner();
- if (oldOwner != null && !oldOwner.equals(owner)) {
- result.put(OWNER_SWITCHES_KEY, AttributeValueUpdate.builder().value(DynamoUtils.createAttributeValue(1L)).action(AttributeAction.ADD).build());
- }
-
- return result;
- }
-
- @Override
- public Map getDynamoEvictLeaseUpdate(final Lease lease) {
- Map result = new HashMap<>();
- AttributeValue value = null;
-
- result.put(LEASE_OWNER_KEY, AttributeValueUpdate.builder().value(value).action(AttributeAction.DELETE).build());
-
- return result;
- }
-
- private AttributeValueUpdate putUpdate(AttributeValue attributeValue) {
- return AttributeValueUpdate.builder().value(attributeValue).action(AttributeAction.PUT).build();
- }
-
- @Override
- public Map getDynamoUpdateLeaseUpdate(final Lease lease) {
- Map result = new HashMap<>();
- result.put(CHECKPOINT_SEQUENCE_NUMBER_KEY, putUpdate(DynamoUtils.createAttributeValue(lease.checkpoint().sequenceNumber())));
- result.put(CHECKPOINT_SUBSEQUENCE_NUMBER_KEY, putUpdate(DynamoUtils.createAttributeValue(lease.checkpoint().subSequenceNumber())));
- result.put(OWNER_SWITCHES_KEY, putUpdate(DynamoUtils.createAttributeValue(lease.ownerSwitchesSinceCheckpoint())));
-
- if (lease.pendingCheckpoint() != null && !lease.pendingCheckpoint().sequenceNumber().isEmpty()) {
- result.put(PENDING_CHECKPOINT_SEQUENCE_KEY, putUpdate(DynamoUtils.createAttributeValue(lease.pendingCheckpoint().sequenceNumber())));
- result.put(PENDING_CHECKPOINT_SUBSEQUENCE_KEY, putUpdate(DynamoUtils.createAttributeValue(lease.pendingCheckpoint().subSequenceNumber())));
- } else {
- result.put(PENDING_CHECKPOINT_SEQUENCE_KEY, AttributeValueUpdate.builder().action(AttributeAction.DELETE).build());
- result.put(PENDING_CHECKPOINT_SUBSEQUENCE_KEY, AttributeValueUpdate.builder().action(AttributeAction.DELETE).build());
- }
- return result;
- }
-
- @Override
- public Collection getKeySchema() {
- List keySchema = new ArrayList<>();
- keySchema.add(KeySchemaElement.builder().attributeName(LEASE_KEY_KEY).keyType(KeyType.HASH).build());
-
- return keySchema;
- }
-
- @Override
- public Collection getAttributeDefinitions() {
- List definitions = new ArrayList<>();
- definitions.add(AttributeDefinition.builder().attributeName(LEASE_KEY_KEY)
- .attributeType(ScalarAttributeType.S).build());
-
- return definitions;
- }
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseTaker.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseTaker.java
deleted file mode 100644
index 165ad01d..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseTaker.java
+++ /dev/null
@@ -1,541 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-package software.amazon.kinesis.leases.dynamodb;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.concurrent.Callable;
-import java.util.concurrent.TimeUnit;
-
-import lombok.extern.slf4j.Slf4j;
-import software.amazon.awssdk.services.cloudwatch.model.StandardUnit;
-import software.amazon.kinesis.annotations.KinesisClientInternalApi;
-import software.amazon.kinesis.leases.Lease;
-import software.amazon.kinesis.leases.LeaseRefresher;
-import software.amazon.kinesis.leases.LeaseTaker;
-import software.amazon.kinesis.leases.exceptions.DependencyException;
-import software.amazon.kinesis.leases.exceptions.InvalidStateException;
-import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException;
-import software.amazon.kinesis.metrics.MetricsFactory;
-import software.amazon.kinesis.metrics.MetricsScope;
-import software.amazon.kinesis.metrics.MetricsLevel;
-import software.amazon.kinesis.metrics.MetricsUtil;
-
-/**
- * An implementation of {@link LeaseTaker} that uses DynamoDB via {@link LeaseRefresher}.
- */
-@Slf4j
-@KinesisClientInternalApi
-public class DynamoDBLeaseTaker implements LeaseTaker {
- private static final int TAKE_RETRIES = 3;
- private static final int SCAN_RETRIES = 1;
-
- // See note on TAKE_LEASES_DIMENSION(Callable) for why we have this callable.
- private static final Callable SYSTEM_CLOCK_CALLABLE = System::nanoTime;
-
- private static final String TAKE_LEASES_DIMENSION = "TakeLeases";
-
- private final LeaseRefresher leaseRefresher;
- private final String workerIdentifier;
- private final long leaseDurationNanos;
- private final MetricsFactory metricsFactory;
-
- private final Map allLeases = new HashMap<>();
- // TODO: Remove these defaults and use the defaults in the config
- private int maxLeasesForWorker = Integer.MAX_VALUE;
- private int maxLeasesToStealAtOneTime = 1;
-
- private long lastScanTimeNanos = 0L;
-
- public DynamoDBLeaseTaker(LeaseRefresher leaseRefresher, String workerIdentifier, long leaseDurationMillis,
- final MetricsFactory metricsFactory) {
- this.leaseRefresher = leaseRefresher;
- this.workerIdentifier = workerIdentifier;
- this.leaseDurationNanos = TimeUnit.MILLISECONDS.toNanos(leaseDurationMillis);
- this.metricsFactory = metricsFactory;
- }
-
- /**
- * Worker will not acquire more than the specified max number of leases even if there are more
- * shards that need to be processed. This can be used in scenarios where a worker is resource constrained or
- * to prevent lease thrashing when small number of workers pick up all leases for small amount of time during
- * deployment.
- * Note that setting a low value may cause data loss (e.g. if there aren't enough Workers to make progress on all
- * shards). When setting the value for this property, one must ensure enough workers are present to process
- * shards and should consider future resharding, child shards that may be blocked on parent shards, some workers
- * becoming unhealthy, etc.
- *
- * @param maxLeasesForWorker Max leases this Worker can handle at a time
- * @return LeaseTaker
- */
- public DynamoDBLeaseTaker withMaxLeasesForWorker(int maxLeasesForWorker) {
- if (maxLeasesForWorker <= 0) {
- throw new IllegalArgumentException("maxLeasesForWorker should be >= 1");
- }
- this.maxLeasesForWorker = maxLeasesForWorker;
- return this;
- }
-
- /**
- * Max leases to steal from a more loaded Worker at one time (for load balancing).
- * Setting this to a higher number can allow for faster load convergence (e.g. during deployments, cold starts),
- * but can cause higher churn in the system.
- *
- * @param maxLeasesToStealAtOneTime Steal up to this many leases at one time (for load balancing)
- * @return LeaseTaker
- */
- public DynamoDBLeaseTaker withMaxLeasesToStealAtOneTime(int maxLeasesToStealAtOneTime) {
- if (maxLeasesToStealAtOneTime <= 0) {
- throw new IllegalArgumentException("maxLeasesToStealAtOneTime should be >= 1");
- }
- this.maxLeasesToStealAtOneTime = maxLeasesToStealAtOneTime;
- return this;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public Map takeLeases() throws DependencyException, InvalidStateException {
- return takeLeases(SYSTEM_CLOCK_CALLABLE);
- }
-
- /**
- * Internal implementation of TAKE_LEASES_DIMENSION. Takes a callable that can provide the time to enable test cases
- * without Thread.sleep. Takes a callable instead of a raw time value because the time needs to be computed as-of
- * immediately after the scan.
- *
- * @param timeProvider
- * Callable that will supply the time
- *
- * @return map of lease key to taken lease
- *
- * @throws DependencyException
- * @throws InvalidStateException
- */
- synchronized Map takeLeases(Callable timeProvider)
- throws DependencyException, InvalidStateException {
- // Key is leaseKey
- Map takenLeases = new HashMap<>();
-
- final MetricsScope scope = MetricsUtil.createMetricsWithOperation(metricsFactory, TAKE_LEASES_DIMENSION);
-
- long startTime = System.currentTimeMillis();
- boolean success = false;
-
- ProvisionedThroughputException lastException = null;
-
- try {
- try {
- for (int i = 1; i <= SCAN_RETRIES; i++) {
- try {
- updateAllLeases(timeProvider);
- success = true;
- } catch (ProvisionedThroughputException e) {
- log.info("Worker {} could not find expired leases on try {} out of {}", workerIdentifier, i,
- TAKE_RETRIES);
- lastException = e;
- }
- }
- } finally {
- MetricsUtil.addWorkerIdentifier(scope, workerIdentifier);
- MetricsUtil.addSuccessAndLatency(scope, "ListLeases", success, startTime, MetricsLevel.DETAILED);
- }
-
- if (lastException != null) {
- log.error("Worker {} could not scan leases table, aborting TAKE_LEASES_DIMENSION. Exception caught by"
- + " last retry:", workerIdentifier, lastException);
- return takenLeases;
- }
-
- List expiredLeases = getExpiredLeases();
-
- Set leasesToTake = computeLeasesToTake(expiredLeases);
- Set untakenLeaseKeys = new HashSet<>();
-
- for (Lease lease : leasesToTake) {
- String leaseKey = lease.leaseKey();
-
- startTime = System.currentTimeMillis();
- success = false;
- try {
- for (int i = 1; i <= TAKE_RETRIES; i++) {
- try {
- if (leaseRefresher.takeLease(lease, workerIdentifier)) {
- lease.lastCounterIncrementNanos(System.nanoTime());
- takenLeases.put(leaseKey, lease);
- } else {
- untakenLeaseKeys.add(leaseKey);
- }
-
- success = true;
- break;
- } catch (ProvisionedThroughputException e) {
- log.info("Could not take lease with key {} for worker {} on try {} out of {} due to"
- + " capacity", leaseKey, workerIdentifier, i, TAKE_RETRIES);
- }
- }
- } finally {
- MetricsUtil.addSuccessAndLatency(scope, "TakeLease", success, startTime, MetricsLevel.DETAILED);
- }
- }
-
- if (takenLeases.size() > 0) {
- log.info("Worker {} successfully took {} leases: {}", workerIdentifier, takenLeases.size(),
- stringJoin(takenLeases.keySet(), ", "));
- }
-
- if (untakenLeaseKeys.size() > 0) {
- log.info("Worker {} failed to take {} leases: {}", workerIdentifier, untakenLeaseKeys.size(),
- stringJoin(untakenLeaseKeys, ", "));
- }
-
- scope.addData("TakenLeases", takenLeases.size(), StandardUnit.COUNT, MetricsLevel.SUMMARY);
- } finally {
- MetricsUtil.endScope(scope);
- }
-
- return takenLeases;
- }
-
- /** Package access for testing purposes.
- *
- * @param strings
- * @param delimiter
- * @return Joined string.
- */
- static String stringJoin(Collection strings, String delimiter) {
- StringBuilder builder = new StringBuilder();
- boolean needDelimiter = false;
- for (String string : strings) {
- if (needDelimiter) {
- builder.append(delimiter);
- }
- builder.append(string);
- needDelimiter = true;
- }
-
- return builder.toString();
- }
-
- /**
- * Scan all leases and update lastRenewalTime. Add new leases and delete old leases.
- *
- * @param timeProvider callable that supplies the current time
- *
- * @return list of expired leases, possibly empty, never null.
- *
- * @throws ProvisionedThroughputException if listLeases fails due to lack of provisioned throughput
- * @throws InvalidStateException if the lease table does not exist
- * @throws DependencyException if listLeases fails in an unexpected way
- */
- private void updateAllLeases(Callable timeProvider)
- throws DependencyException, InvalidStateException, ProvisionedThroughputException {
- List freshList = leaseRefresher.listLeases();
- try {
- lastScanTimeNanos = timeProvider.call();
- } catch (Exception e) {
- throw new DependencyException("Exception caught from timeProvider", e);
- }
-
- // This set will hold the lease keys not updated by the previous listLeases call.
- Set notUpdated = new HashSet<>(allLeases.keySet());
-
- // Iterate over all leases, finding ones to try to acquire that haven't changed since the last iteration
- for (Lease lease : freshList) {
- String leaseKey = lease.leaseKey();
-
- Lease oldLease = allLeases.get(leaseKey);
- allLeases.put(leaseKey, lease);
- notUpdated.remove(leaseKey);
-
- if (oldLease != null) {
- // If we've seen this lease before...
- if (oldLease.leaseCounter().equals(lease.leaseCounter())) {
- // ...and the counter hasn't changed, propagate the lastRenewalNanos time from the old lease
- lease.lastCounterIncrementNanos(oldLease.lastCounterIncrementNanos());
- } else {
- // ...and the counter has changed, set lastRenewalNanos to the time of the scan.
- lease.lastCounterIncrementNanos(lastScanTimeNanos);
- }
- } else {
- if (lease.leaseOwner() == null) {
- // if this new lease is unowned, it's never been renewed.
- lease.lastCounterIncrementNanos(0L);
-
- if (log.isDebugEnabled()) {
- log.debug("Treating new lease with key {} as never renewed because it is new and unowned.",
- leaseKey);
- }
- } else {
- // if this new lease is owned, treat it as renewed as of the scan
- lease.lastCounterIncrementNanos(lastScanTimeNanos);
- if (log.isDebugEnabled()) {
- log.debug("Treating new lease with key {} as recently renewed because it is new and owned.",
- leaseKey);
- }
- }
- }
- }
-
- // Remove dead leases from allLeases
- for (String key : notUpdated) {
- allLeases.remove(key);
- }
- }
-
- /**
- * @return list of leases that were expired as of our last scan.
- */
- private List getExpiredLeases() {
- List expiredLeases = new ArrayList<>();
-
- for (Lease lease : allLeases.values()) {
- if (lease.isExpired(leaseDurationNanos, lastScanTimeNanos)) {
- expiredLeases.add(lease);
- }
- }
-
- return expiredLeases;
- }
-
- /**
- * Compute the number of leases I should try to take based on the state of the system.
- *
- * @param expiredLeases list of leases we determined to be expired
- * @return set of leases to take.
- */
- private Set computeLeasesToTake(List expiredLeases) {
- Map leaseCounts = computeLeaseCounts(expiredLeases);
- Set leasesToTake = new HashSet<>();
- final MetricsScope scope = MetricsUtil.createMetricsWithOperation(metricsFactory, TAKE_LEASES_DIMENSION);
- MetricsUtil.addWorkerIdentifier(scope, workerIdentifier);
-
- try {
- int numLeases = allLeases.size();
- int numWorkers = leaseCounts.size();
-
- if (numLeases == 0) {
- // If there are no leases, I shouldn't try to take any.
- return leasesToTake;
- }
-
- int target;
- if (numWorkers >= numLeases) {
- // If we have n leases and n or more workers, each worker can have up to 1 lease, including myself.
- target = 1;
- } else {
- /*
- * numWorkers must be < numLeases.
- *
- * Our target for each worker is numLeases / numWorkers (+1 if numWorkers doesn't evenly divide numLeases)
- */
- target = numLeases / numWorkers + (numLeases % numWorkers == 0 ? 0 : 1);
-
- // Spill over is the number of leases this worker should have claimed, but did not because it would
- // exceed the max allowed for this worker.
- int leaseSpillover = Math.max(0, target - maxLeasesForWorker);
- if (target > maxLeasesForWorker) {
- log.warn(
- "Worker {} target is {} leases and maxLeasesForWorker is {}. Resetting target to {},"
- + " lease spillover is {}. Note that some shards may not be processed if no other "
- + "workers are able to pick them up.",
- workerIdentifier, target, maxLeasesForWorker, maxLeasesForWorker, leaseSpillover);
- target = maxLeasesForWorker;
- }
- scope.addData("LeaseSpillover", leaseSpillover, StandardUnit.COUNT, MetricsLevel.SUMMARY);
- }
-
- int myCount = leaseCounts.get(workerIdentifier);
- int numLeasesToReachTarget = target - myCount;
-
- if (numLeasesToReachTarget <= 0) {
- // If we don't need anything, return the empty set.
- return leasesToTake;
- }
-
- // Shuffle expiredLeases so workers don't all try to contend for the same leases.
- Collections.shuffle(expiredLeases);
-
- int originalExpiredLeasesSize = expiredLeases.size();
- if (expiredLeases.size() > 0) {
- // If we have expired leases, get up to leases from expiredLeases
- for (; numLeasesToReachTarget > 0 && expiredLeases.size() > 0; numLeasesToReachTarget--) {
- leasesToTake.add(expiredLeases.remove(0));
- }
- } else {
- // If there are no expired leases and we need a lease, consider stealing.
- List leasesToSteal = chooseLeasesToSteal(leaseCounts, numLeasesToReachTarget, target);
- for (Lease leaseToSteal : leasesToSteal) {
- log.info("Worker {} needed {} leases but none were expired, so it will steal lease {} from {}",
- workerIdentifier, numLeasesToReachTarget, leaseToSteal.leaseKey(),
- leaseToSteal.leaseOwner());
- leasesToTake.add(leaseToSteal);
- }
- }
-
- if (!leasesToTake.isEmpty()) {
- log.info(
- "Worker {} saw {} total leases, {} available leases, {} "
- + "workers. Target is {} leases, I have {} leases, I will take {} leases",
- workerIdentifier, numLeases, originalExpiredLeasesSize, numWorkers, target, myCount,
- leasesToTake.size());
- }
-
- scope.addData("TotalLeases", numLeases, StandardUnit.COUNT, MetricsLevel.DETAILED);
- scope.addData("ExpiredLeases", originalExpiredLeasesSize, StandardUnit.COUNT, MetricsLevel.SUMMARY);
- scope.addData("NumWorkers", numWorkers, StandardUnit.COUNT, MetricsLevel.SUMMARY);
- scope.addData("NeededLeases", numLeasesToReachTarget, StandardUnit.COUNT, MetricsLevel.DETAILED);
- scope.addData("LeasesToTake", leasesToTake.size(), StandardUnit.COUNT, MetricsLevel.DETAILED);
- } finally {
- MetricsUtil.endScope(scope);
- }
-
- return leasesToTake;
- }
-
- /**
- * Choose leases to steal by randomly selecting one or more (up to max) from the most loaded worker.
- * Stealing rules:
- *
- * Steal up to maxLeasesToStealAtOneTime leases from the most loaded worker if
- * a) he has > target leases and I need >= 1 leases : steal min(leases needed, maxLeasesToStealAtOneTime)
- * b) he has == target leases and I need > 1 leases : steal 1
- *
- * @param leaseCounts map of workerIdentifier to lease count
- * @param needed # of leases needed to reach the target leases for the worker
- * @param target target # of leases per worker
- * @return Leases to steal, or empty list if we should not steal
- */
- private List chooseLeasesToSteal(Map leaseCounts, int needed, int target) {
- List leasesToSteal = new ArrayList<>();
-
- Entry mostLoadedWorker = null;
- // Find the most loaded worker
- for (Entry worker : leaseCounts.entrySet()) {
- if (mostLoadedWorker == null || mostLoadedWorker.getValue() < worker.getValue()) {
- mostLoadedWorker = worker;
- }
- }
-
- int numLeasesToSteal = 0;
- if ((mostLoadedWorker.getValue() >= target) && (needed > 0)) {
- int leasesOverTarget = mostLoadedWorker.getValue() - target;
- numLeasesToSteal = Math.min(needed, leasesOverTarget);
- // steal 1 if we need > 1 and max loaded worker has target leases.
- if ((needed > 1) && (numLeasesToSteal == 0)) {
- numLeasesToSteal = 1;
- }
- numLeasesToSteal = Math.min(numLeasesToSteal, maxLeasesToStealAtOneTime);
- }
-
- if (numLeasesToSteal <= 0) {
- if (log.isDebugEnabled()) {
- log.debug(String.format("Worker %s not stealing from most loaded worker %s. He has %d,"
- + " target is %d, and I need %d",
- workerIdentifier,
- mostLoadedWorker.getKey(),
- mostLoadedWorker.getValue(),
- target,
- needed));
- }
- return leasesToSteal;
- } else {
- if (log.isDebugEnabled()) {
- log.debug("Worker {} will attempt to steal {} leases from most loaded worker {}. "
- + " He has {} leases, target is {}, I need {}, maxLeasesToSteatAtOneTime is {}.",
- workerIdentifier,
- numLeasesToSteal,
- mostLoadedWorker.getKey(),
- mostLoadedWorker.getValue(),
- target,
- needed,
- maxLeasesToStealAtOneTime);
- }
- }
-
- String mostLoadedWorkerIdentifier = mostLoadedWorker.getKey();
- List candidates = new ArrayList<>();
- // Collect leases belonging to that worker
- for (Lease lease : allLeases.values()) {
- if (mostLoadedWorkerIdentifier.equals(lease.leaseOwner())) {
- candidates.add(lease);
- }
- }
-
- // Return random ones
- Collections.shuffle(candidates);
- int toIndex = Math.min(candidates.size(), numLeasesToSteal);
- leasesToSteal.addAll(candidates.subList(0, toIndex));
-
- return leasesToSteal;
- }
-
- /**
- * Count leases by host. Always includes myself, but otherwise only includes hosts that are currently holding
- * leases.
- *
- * @param expiredLeases list of leases that are currently expired
- * @return map of workerIdentifier to lease count
- */
- private Map computeLeaseCounts(List expiredLeases) {
- Map leaseCounts = new HashMap<>();
-
- // Compute the number of leases per worker by looking through allLeases and ignoring leases that have expired.
- for (Lease lease : allLeases.values()) {
- if (!expiredLeases.contains(lease)) {
- String leaseOwner = lease.leaseOwner();
- Integer oldCount = leaseCounts.get(leaseOwner);
- if (oldCount == null) {
- leaseCounts.put(leaseOwner, 1);
- } else {
- leaseCounts.put(leaseOwner, oldCount + 1);
- }
- }
- }
-
- // If I have no leases, I wasn't represented in leaseCounts. Let's fix that.
- Integer myCount = leaseCounts.get(workerIdentifier);
- if (myCount == null) {
- myCount = 0;
- leaseCounts.put(workerIdentifier, myCount);
- }
-
- return leaseCounts;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public String getWorkerIdentifier() {
- return workerIdentifier;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public synchronized List allLeases() {
- return new ArrayList<>(allLeases.values());
- }
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/TableConstants.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/TableConstants.java
deleted file mode 100644
index 3848c2f0..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/TableConstants.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-
-package software.amazon.kinesis.leases.dynamodb;
-
-import lombok.AccessLevel;
-import lombok.NoArgsConstructor;
-
-/**
- * This class is just a holder for initial lease table IOPs units. This class will be removed in a future release.
- */
-@Deprecated
-@NoArgsConstructor(access = AccessLevel.PRIVATE)
-public class TableConstants {
- public static final long DEFAULT_INITIAL_LEASE_TABLE_READ_CAPACITY = 10L;
- public static final long DEFAULT_INITIAL_LEASE_TABLE_WRITE_CAPACITY = 10L;
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/TableCreatorCallback.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/TableCreatorCallback.java
deleted file mode 100644
index 088ba924..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/TableCreatorCallback.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-
-package software.amazon.kinesis.leases.dynamodb;
-
-/**
- * Callback interface for interacting with the DynamoDB lease table post creation.
- */
-@FunctionalInterface
-public interface TableCreatorCallback {
- /**
- * NoOp implemetation for TableCreatorCallback
- */
- TableCreatorCallback NOOP_TABLE_CREATOR_CALLBACK = (TableCreatorCallbackInput tableCreatorCallbackInput) -> {
- // Do nothing
- };
-
- /**
- * Actions needed to be performed on the DynamoDB lease table once the table has been created and is in the ACTIVE
- * status. Will not be called if the table previously exists.
- *
- * @param tableCreatorCallbackInput
- * Input object for table creator
- */
- void performAction(TableCreatorCallbackInput tableCreatorCallbackInput);
-
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/TableCreatorCallbackInput.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/TableCreatorCallbackInput.java
deleted file mode 100644
index 4c4d6f12..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/TableCreatorCallbackInput.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-
-package software.amazon.kinesis.leases.dynamodb;
-
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.NonNull;
-import lombok.ToString;
-import lombok.experimental.Accessors;
-import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
-
-/**
- *
- */
-@Builder(toBuilder = true)
-@Data
-@Accessors(fluent = true)
-@ToString
-@EqualsAndHashCode
-public class TableCreatorCallbackInput {
- @NonNull
- private final DynamoDbAsyncClient dynamoDbClient;
- @NonNull
- private final String tableName;
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/exceptions/DependencyException.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/exceptions/DependencyException.java
deleted file mode 100644
index efaa1ad9..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/exceptions/DependencyException.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-package software.amazon.kinesis.leases.exceptions;
-
-/**
- * Indicates that a lease operation has failed because a dependency of the leasing system has failed. This will happen
- * if DynamoDB throws an InternalServerException or a generic AmazonClientException (the specific subclasses of
- * AmazonClientException are all handled more gracefully).
- */
-public class DependencyException extends LeasingException {
-
- private static final long serialVersionUID = 1L;
-
- public DependencyException(Throwable e) {
- super(e);
- }
-
- public DependencyException(String message, Throwable e) {
- super(message, e);
- }
-
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/exceptions/InvalidStateException.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/exceptions/InvalidStateException.java
deleted file mode 100644
index 0929fee2..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/exceptions/InvalidStateException.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-package software.amazon.kinesis.leases.exceptions;
-
-/**
- * Indicates that a lease operation has failed because DynamoDB is an invalid state. The most common example is failing
- * to create the DynamoDB table before doing any lease operations.
- */
-public class InvalidStateException extends LeasingException {
-
- private static final long serialVersionUID = 1L;
-
- public InvalidStateException(Throwable e) {
- super(e);
- }
-
- public InvalidStateException(String message, Throwable e) {
- super(message, e);
- }
-
- public InvalidStateException(String message) {
- super(message);
- }
-
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/exceptions/LeasingException.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/exceptions/LeasingException.java
deleted file mode 100644
index a59e69c1..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/exceptions/LeasingException.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-package software.amazon.kinesis.leases.exceptions;
-
-/**
- * Top-level exception type for all exceptions thrown by the leasing code.
- */
-public class LeasingException extends Exception {
-
- public LeasingException(Throwable e) {
- super(e);
- }
-
- public LeasingException(String message, Throwable e) {
- super(message, e);
- }
-
- public LeasingException(String message) {
- super(message);
- }
-
- private static final long serialVersionUID = 1L;
-
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/exceptions/ProvisionedThroughputException.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/exceptions/ProvisionedThroughputException.java
deleted file mode 100644
index 9409d3db..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/exceptions/ProvisionedThroughputException.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-package software.amazon.kinesis.leases.exceptions;
-
-/**
- * Indicates that a lease operation has failed due to lack of provisioned throughput for a DynamoDB table.
- */
-public class ProvisionedThroughputException extends LeasingException {
-
- private static final long serialVersionUID = 1L;
-
- public ProvisionedThroughputException(Throwable e) {
- super(e);
- }
-
- public ProvisionedThroughputException(String message, Throwable e) {
- super(message, e);
- }
-
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/exceptions/ShardSyncer.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/exceptions/ShardSyncer.java
deleted file mode 100644
index 4e9245f6..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/exceptions/ShardSyncer.java
+++ /dev/null
@@ -1,46 +0,0 @@
-package software.amazon.kinesis.leases.exceptions;
-
-import lombok.NonNull;
-import software.amazon.kinesis.common.InitialPositionInStreamExtended;
-import software.amazon.kinesis.exceptions.internal.KinesisClientLibIOException;
-import software.amazon.kinesis.leases.HierarchicalShardSyncer;
-import software.amazon.kinesis.leases.LeaseRefresher;
-import software.amazon.kinesis.leases.ShardDetector;
-import software.amazon.kinesis.metrics.MetricsScope;
-
-/**
- * Helper class to sync leases with shards of the Kinesis stream.
- * It will create new leases/activities when it discovers new Kinesis shards (bootstrap/resharding).
- * It deletes leases for shards that have been trimmed from Kinesis, or if we've completed processing it
- * and begun processing it's child shards.
- *
- * NOTE: This class is deprecated and will be removed in a future release.
- */
-@Deprecated
-public class ShardSyncer {
- private static final HierarchicalShardSyncer HIERARCHICAL_SHARD_SYNCER = new HierarchicalShardSyncer();
-
- /**
- * NOTE: This method is deprecated and will be removed in a future release.
- *
- * @param shardDetector
- * @param leaseRefresher
- * @param initialPosition
- * @param cleanupLeasesOfCompletedShards
- * @param ignoreUnexpectedChildShards
- * @param scope
- * @throws DependencyException
- * @throws InvalidStateException
- * @throws ProvisionedThroughputException
- * @throws KinesisClientLibIOException
- */
- @Deprecated
- public static synchronized void checkAndCreateLeasesForNewShards(@NonNull final ShardDetector shardDetector,
- final LeaseRefresher leaseRefresher, final InitialPositionInStreamExtended initialPosition,
- final boolean cleanupLeasesOfCompletedShards, final boolean ignoreUnexpectedChildShards,
- final MetricsScope scope) throws DependencyException, InvalidStateException, ProvisionedThroughputException,
- KinesisClientLibIOException {
- HIERARCHICAL_SHARD_SYNCER.checkAndCreateLeaseForNewShards(shardDetector, leaseRefresher, initialPosition,
- cleanupLeasesOfCompletedShards, ignoreUnexpectedChildShards, scope);
- }
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/BlockOnParentShardTask.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/BlockOnParentShardTask.java
deleted file mode 100644
index e14b111a..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/BlockOnParentShardTask.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-package software.amazon.kinesis.lifecycle;
-
-import lombok.AccessLevel;
-import lombok.NonNull;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import software.amazon.kinesis.annotations.KinesisClientInternalApi;
-import software.amazon.kinesis.exceptions.internal.BlockedOnParentShardException;
-import software.amazon.kinesis.leases.Lease;
-import software.amazon.kinesis.leases.LeaseRefresher;
-import software.amazon.kinesis.leases.ShardInfo;
-import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber;
-
-/**
- * Task to block until processing of all data records in the parent shard(s) is completed.
- * We check if we have checkpoint(s) for the parent shard(s).
- * If a checkpoint for a parent shard is found, we poll and wait until the checkpoint value is SHARD_END
- * (application has checkpointed after processing all records in the shard).
- * If we don't find a checkpoint for the parent shard(s), we assume they have been trimmed and directly
- * proceed with processing data from the shard.
- */
-@RequiredArgsConstructor(access = AccessLevel.PACKAGE)
-@Slf4j
-@KinesisClientInternalApi
-// TODO: Check for non null values
-public class BlockOnParentShardTask implements ConsumerTask {
- @NonNull
- private final ShardInfo shardInfo;
- private final LeaseRefresher leaseRefresher;
- // Sleep for this duration if the parent shards have not completed processing, or we encounter an exception.
- private final long parentShardPollIntervalMillis;
-
- private final TaskType taskType = TaskType.BLOCK_ON_PARENT_SHARDS;
-
- /*
- * (non-Javadoc)
- *
- * @see com.amazonaws.services.kinesis.clientlibrary.lib.worker.ConsumerTask#call()
- */
- @Override
- public TaskResult call() {
- Exception exception = null;
-
- try {
- boolean blockedOnParentShard = false;
- for (String shardId : shardInfo.parentShardIds()) {
- Lease lease = leaseRefresher.getLease(shardId);
- if (lease != null) {
- ExtendedSequenceNumber checkpoint = lease.checkpoint();
- if ((checkpoint == null) || (!checkpoint.equals(ExtendedSequenceNumber.SHARD_END))) {
- log.debug("Shard {} is not yet done. Its current checkpoint is {}", shardId, checkpoint);
- blockedOnParentShard = true;
- exception = new BlockedOnParentShardException("Parent shard not yet done");
- break;
- } else {
- log.debug("Shard {} has been completely processed.", shardId);
- }
- } else {
- log.info("No lease found for shard {}. Not blocking on completion of this shard.", shardId);
- }
- }
-
- if (!blockedOnParentShard) {
- log.info("No need to block on parents {} of shard {}", shardInfo.parentShardIds(), shardInfo.shardId());
- return new TaskResult(null);
- }
- } catch (Exception e) {
- log.error("Caught exception when checking for parent shard checkpoint", e);
- exception = e;
- }
- try {
- Thread.sleep(parentShardPollIntervalMillis);
- } catch (InterruptedException e) {
- log.error("Sleep interrupted when waiting on parent shard(s) of {}", shardInfo.shardId(), e);
- }
-
- return new TaskResult(exception);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see com.amazonaws.services.kinesis.clientlibrary.lib.worker.ConsumerTask#taskType()
- */
- @Override
- public TaskType taskType() {
- return taskType;
- }
-
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ConsumerState.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ConsumerState.java
deleted file mode 100644
index cf246b28..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ConsumerState.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-
-package software.amazon.kinesis.lifecycle;
-
-import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput;
-
-/**
- * Represents a the current state of the consumer. This handles the creation of tasks for the consumer, and what to
- * do when a transition occurs.
- *
- */
-interface ConsumerState {
- /**
- * Creates a new task for this state using the passed in consumer to build the task. If there is no task
- * required for this state it may return a null value. {@link ConsumerState}'s are allowed to modify the
- * consumer during the execution of this method.
- *
- * @param consumerArgument
- * configuration specific to the task being created
- * @param consumer
- * the consumer to use build the task, or execute state.
- * @param input
- * the process input received, this may be null if it's a control message
- * @return a valid task for this state or null if there is no task required.
- */
- ConsumerTask createTask(ShardConsumerArgument consumerArgument, ShardConsumer consumer, ProcessRecordsInput input);
-
- /**
- * Provides the next state of the consumer upon success of the task return by
- * {@link ConsumerState#createTask(ShardConsumerArgument, ShardConsumer, ProcessRecordsInput)}.
- *
- * @return the next state that the consumer should transition to, this may be the same object as the current
- * state.
- */
- ConsumerState successTransition();
-
- /**
- * Provides the next state of the consumer if the task failed. This defaults to no state change.
- *
- * @return the state to change to upon a task failure
- */
- default ConsumerState failureTransition() {
- return this;
- }
-
- /**
- * Provides the next state of the consumer when a shutdown has been requested. The returned state is dependent
- * on the current state, and the shutdown reason.
- *
- * @param shutdownReason
- * the reason that a shutdown was requested
- * @return the next state that the consumer should transition to, this may be the same object as the current
- * state.
- */
- ConsumerState shutdownTransition(ShutdownReason shutdownReason);
-
- /**
- * The type of task that {@link ConsumerState#createTask(ShardConsumerArgument, ShardConsumer, ProcessRecordsInput)}
- * would return. This is always a valid state
- * even if createTask would return a null value.
- *
- * @return the type of task that this state represents.
- */
- TaskType taskType();
-
- /**
- * An enumeration represent the type of this state. Different consumer states may return the same
- * {@link ConsumerStates.ShardConsumerState}.
- *
- * @return the type of consumer state this represents.
- */
- ConsumerStates.ShardConsumerState state();
-
- boolean isTerminal();
-
- /**
- * Whether this state requires data to be available before the task can be created
- *
- * @return true if the task requires data to be available before creation, false otherwise
- */
- default boolean requiresDataAvailability() {
- return false;
- }
-
- /**
- * Indicates whether a state requires an external event to re-awaken for processing.
- *
- * @return true if the state is some external event to restart processing, false if events can be immediately
- * dispatched.
- */
- default boolean requiresAwake() {
- return false;
- }
-
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/InitializeTask.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/InitializeTask.java
deleted file mode 100644
index 4d151574..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/InitializeTask.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-package software.amazon.kinesis.lifecycle;
-
-import lombok.NonNull;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import software.amazon.kinesis.annotations.KinesisClientInternalApi;
-import software.amazon.kinesis.checkpoint.Checkpoint;
-import software.amazon.kinesis.checkpoint.ShardRecordProcessorCheckpointer;
-import software.amazon.kinesis.common.InitialPositionInStreamExtended;
-import software.amazon.kinesis.leases.ShardInfo;
-import software.amazon.kinesis.lifecycle.events.InitializationInput;
-import software.amazon.kinesis.metrics.MetricsFactory;
-import software.amazon.kinesis.metrics.MetricsLevel;
-import software.amazon.kinesis.metrics.MetricsScope;
-import software.amazon.kinesis.metrics.MetricsUtil;
-import software.amazon.kinesis.processor.Checkpointer;
-import software.amazon.kinesis.processor.ShardRecordProcessor;
-import software.amazon.kinesis.retrieval.RecordsPublisher;
-import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber;
-
-/**
- * Task for initializing shard position and invoking the ShardRecordProcessor initialize() API.
- */
-@RequiredArgsConstructor
-@Slf4j
-@KinesisClientInternalApi
-public class InitializeTask implements ConsumerTask {
- private static final String INITIALIZE_TASK_OPERATION = "InitializeTask";
- private static final String RECORD_PROCESSOR_INITIALIZE_METRIC = "RecordProcessor.initialize";
-
- @NonNull
- private final ShardInfo shardInfo;
- @NonNull
- private final ShardRecordProcessor shardRecordProcessor;
- @NonNull
- private final Checkpointer checkpoint;
- @NonNull
- private final ShardRecordProcessorCheckpointer recordProcessorCheckpointer;
- @NonNull
- private final InitialPositionInStreamExtended initialPositionInStream;
- @NonNull
- private final RecordsPublisher cache;
-
- // Back off for this interval if we encounter a problem (exception)
- private final long backoffTimeMillis;
- @NonNull
- private final MetricsFactory metricsFactory;
-
- private final TaskType taskType = TaskType.INITIALIZE;
-
- /*
- * Initializes the data fetcher (position in shard) and invokes the ShardRecordProcessor initialize() API.
- * (non-Javadoc)
- *
- * @see com.amazonaws.services.kinesis.clientlibrary.lib.worker.ConsumerTask#call()
- */
- @Override
- public TaskResult call() {
- boolean applicationException = false;
- Exception exception = null;
-
- try {
- log.debug("Initializing ShardId {}", shardInfo);
- Checkpoint initialCheckpointObject = checkpoint.getCheckpointObject(shardInfo.shardId());
- ExtendedSequenceNumber initialCheckpoint = initialCheckpointObject.checkpoint();
- log.debug("[{}]: Checkpoint: {} -- Initial Position: {}", shardInfo.shardId(), initialCheckpoint,
- initialPositionInStream);
-
- cache.start(initialCheckpoint, initialPositionInStream);
-
- recordProcessorCheckpointer.largestPermittedCheckpointValue(initialCheckpoint);
- recordProcessorCheckpointer.setInitialCheckpointValue(initialCheckpoint);
-
- log.debug("Calling the record processor initialize().");
- final InitializationInput initializationInput = InitializationInput.builder()
- .shardId(shardInfo.shardId())
- .extendedSequenceNumber(initialCheckpoint)
- .pendingCheckpointSequenceNumber(initialCheckpointObject.pendingCheckpoint())
- .build();
-
- final MetricsScope scope = MetricsUtil.createMetricsWithOperation(metricsFactory,
- INITIALIZE_TASK_OPERATION);
-
- final long startTime = System.currentTimeMillis();
- try {
- shardRecordProcessor.initialize(initializationInput);
- log.debug("Record processor initialize() completed.");
- } catch (Exception e) {
- applicationException = true;
- throw e;
- } finally {
- MetricsUtil.addLatency(scope, RECORD_PROCESSOR_INITIALIZE_METRIC, startTime, MetricsLevel.SUMMARY);
- MetricsUtil.endScope(scope);
- }
-
- return new TaskResult(null);
- } catch (Exception e) {
- if (applicationException) {
- log.error("Application initialize() threw exception: ", e);
- } else {
- log.error("Caught exception: ", e);
- }
- exception = e;
- // backoff if we encounter an exception.
- try {
- Thread.sleep(this.backoffTimeMillis);
- } catch (InterruptedException ie) {
- log.debug("Interrupted sleep", ie);
- }
- }
-
- return new TaskResult(exception);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see com.amazonaws.services.kinesis.clientlibrary.lib.worker.ConsumerTask#taskType()
- */
- @Override
- public TaskType taskType() {
- return taskType;
- }
-
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/LifecycleConfig.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/LifecycleConfig.java
deleted file mode 100644
index b04d75ce..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/LifecycleConfig.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-
-package software.amazon.kinesis.lifecycle;
-
-import java.util.Optional;
-
-import lombok.Data;
-import lombok.experimental.Accessors;
-import software.amazon.kinesis.retrieval.AggregatorUtil;
-
-/**
- * Used by the KCL to configure the lifecycle.
- */
-@Data
-@Accessors(fluent = true)
-public class LifecycleConfig {
- /**
- * Logs warn message if as task is held in a task for more than the set time.
- *
- * Default value: {@link Optional#empty()}
- */
- private Optional logWarningForTaskAfterMillis = Optional.empty();
-
- /**
- * Backoff time in milliseconds for Amazon Kinesis Client Library tasks (in the event of failures).
- *
- * Default value: 500L
- */
- private long taskBackoffTimeMillis = 500L;
-
- /**
- * AggregatorUtil is responsible for deaggregating KPL records.
- */
- private AggregatorUtil aggregatorUtil = new AggregatorUtil();
-
- /**
- * TaskExecutionListener to be used to handle events during task execution lifecycle for a shard.
- *
- * Default value: {@link NoOpTaskExecutionListener}
- */
- private TaskExecutionListener taskExecutionListener = new NoOpTaskExecutionListener();
-}
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/NoOpTaskExecutionListener.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/NoOpTaskExecutionListener.java
deleted file mode 100644
index 95d225fa..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/NoOpTaskExecutionListener.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-package software.amazon.kinesis.lifecycle;
-
-import software.amazon.kinesis.lifecycle.events.TaskExecutionListenerInput;
-
-/**
- * NoOp implementation of {@link TaskExecutionListener} interface that takes no action on task execution.
- */
-public class NoOpTaskExecutionListener implements TaskExecutionListener {
- @Override
- public void beforeTaskExecution(TaskExecutionListenerInput input) {
- }
-
- @Override
- public void afterTaskExecution(TaskExecutionListenerInput input) {
- }
-}
-
diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ProcessTask.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ProcessTask.java
deleted file mode 100644
index f9cdd2ac..00000000
--- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ProcessTask.java
+++ /dev/null
@@ -1,292 +0,0 @@
-/*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-package software.amazon.kinesis.lifecycle;
-
-import java.util.List;
-import java.util.ListIterator;
-
-import lombok.NonNull;
-import lombok.extern.slf4j.Slf4j;
-import software.amazon.awssdk.services.cloudwatch.model.StandardUnit;
-import software.amazon.awssdk.services.kinesis.model.Shard;
-import software.amazon.kinesis.annotations.KinesisClientInternalApi;
-import software.amazon.kinesis.checkpoint.ShardRecordProcessorCheckpointer;
-import software.amazon.kinesis.leases.ShardDetector;
-import software.amazon.kinesis.leases.ShardInfo;
-import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput;
-import software.amazon.kinesis.metrics.MetricsFactory;
-import software.amazon.kinesis.metrics.MetricsScope;
-import software.amazon.kinesis.metrics.MetricsLevel;
-import software.amazon.kinesis.metrics.MetricsUtil;
-import software.amazon.kinesis.processor.ShardRecordProcessor;
-import software.amazon.kinesis.retrieval.AggregatorUtil;
-import software.amazon.kinesis.retrieval.KinesisClientRecord;
-import software.amazon.kinesis.retrieval.ThrottlingReporter;
-import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber;
-
-/**
- * Task for fetching data records and invoking processRecords() on the record processor instance.
- */
-@Slf4j
-@KinesisClientInternalApi
-public class ProcessTask implements ConsumerTask {
- private static final String PROCESS_TASK_OPERATION = "ProcessTask";
- private static final String DATA_BYTES_PROCESSED_METRIC = "DataBytesProcessed";
- private static final String RECORDS_PROCESSED_METRIC = "RecordsProcessed";
- private static final String RECORD_PROCESSOR_PROCESS_RECORDS_METRIC = "RecordProcessor.processRecords";
- private static final String MILLIS_BEHIND_LATEST_METRIC = "MillisBehindLatest";
-
- private final ShardInfo shardInfo;
- private final ShardRecordProcessor shardRecordProcessor;
- private final ShardRecordProcessorCheckpointer recordProcessorCheckpointer;
- private final TaskType taskType = TaskType.PROCESS;
- private final long backoffTimeMillis;
- private final Shard shard;
- private final ThrottlingReporter throttlingReporter;
- private final boolean shouldCallProcessRecordsEvenForEmptyRecordList;
- private final long idleTimeInMilliseconds;
- private final ProcessRecordsInput processRecordsInput;
- private final MetricsFactory metricsFactory;
- private final AggregatorUtil aggregatorUtil;
-
- public ProcessTask(@NonNull ShardInfo shardInfo,
- @NonNull ShardRecordProcessor shardRecordProcessor,
- @NonNull ShardRecordProcessorCheckpointer recordProcessorCheckpointer,
- long backoffTimeMillis,
- boolean skipShardSyncAtWorkerInitializationIfLeasesExist,
- ShardDetector shardDetector,
- @NonNull ThrottlingReporter throttlingReporter,
- ProcessRecordsInput processRecordsInput,
- boolean shouldCallProcessRecordsEvenForEmptyRecordList,
- long idleTimeInMilliseconds,
- @NonNull AggregatorUtil aggregatorUtil,
- @NonNull MetricsFactory metricsFactory) {
- this.shardInfo = shardInfo;
- this.shardRecordProcessor = shardRecordProcessor;
- this.recordProcessorCheckpointer = recordProcessorCheckpointer;
- this.backoffTimeMillis = backoffTimeMillis;
- this.throttlingReporter = throttlingReporter;
- this.processRecordsInput = processRecordsInput;
- this.shouldCallProcessRecordsEvenForEmptyRecordList = shouldCallProcessRecordsEvenForEmptyRecordList;
- this.idleTimeInMilliseconds = idleTimeInMilliseconds;
- this.metricsFactory = metricsFactory;
-
- if (!skipShardSyncAtWorkerInitializationIfLeasesExist) {
- this.shard = shardDetector.shard(shardInfo.shardId());
- } else {
- this.shard = null;
- }
-
- if (this.shard == null && !skipShardSyncAtWorkerInitializationIfLeasesExist) {
- log.warn("Cannot get the shard for this ProcessTask, so duplicate KPL user records "
- + "in the event of resharding will not be dropped during deaggregation of Amazon "
- + "Kinesis records.");
- }
- this.aggregatorUtil = aggregatorUtil;
-
- this.recordProcessorCheckpointer.checkpointer().operation(PROCESS_TASK_OPERATION);
- }
-
- /*
- * (non-Javadoc)
- * @see com.amazonaws.services.kinesis.clientlibrary.lib.worker.ConsumerTask#call()
- */
- @Override
- public TaskResult call() {
- final MetricsScope scope = MetricsUtil.createMetricsWithOperation(metricsFactory, PROCESS_TASK_OPERATION);
- MetricsUtil.addShardId(scope, shardInfo.shardId());
- long startTimeMillis = System.currentTimeMillis();
- boolean success = false;
- try {
- scope.addData(RECORDS_PROCESSED_METRIC, 0, StandardUnit.COUNT, MetricsLevel.SUMMARY);
- scope.addData(DATA_BYTES_PROCESSED_METRIC, 0, StandardUnit.BYTES, MetricsLevel.SUMMARY);
- Exception exception = null;
-
- try {
- if (processRecordsInput.millisBehindLatest() != null) {
- scope.addData(MILLIS_BEHIND_LATEST_METRIC, processRecordsInput.millisBehindLatest(),
- StandardUnit.MILLISECONDS, MetricsLevel.SUMMARY);
- }
-
- if (processRecordsInput.isAtShardEnd() && processRecordsInput.records().isEmpty()) {
- log.info("Reached end of shard {} and have no records to process", shardInfo.shardId());
- return new TaskResult(null, true);
- }
-
- throttlingReporter.success();
- List records = deaggregateAnyKplRecords(processRecordsInput.records());
-
-
- if (!records.isEmpty()) {
- scope.addData(RECORDS_PROCESSED_METRIC, records.size(), StandardUnit.COUNT, MetricsLevel.SUMMARY);
- }
-
- recordProcessorCheckpointer.largestPermittedCheckpointValue(filterAndGetMaxExtendedSequenceNumber(
- scope, records, recordProcessorCheckpointer.lastCheckpointValue(),
- recordProcessorCheckpointer.largestPermittedCheckpointValue()));
-
- if (shouldCallProcessRecords(records)) {
- callProcessRecords(processRecordsInput, records);
- }
- success = true;
- } catch (RuntimeException e) {
- log.error("ShardId {}: Caught exception: ", shardInfo.shardId(), e);
- exception = e;
- backoff();
- }
-
- if (processRecordsInput.isAtShardEnd()) {
- log.info("Reached end of shard {}, and processed {} records", shardInfo.shardId(), processRecordsInput.records().size());
- return new TaskResult(null, true);
- }
- return new TaskResult(exception);
- } finally {
- MetricsUtil.addSuccessAndLatency(scope, success, startTimeMillis, MetricsLevel.SUMMARY);
- MetricsUtil.endScope(scope);
- }
- }
-
- private List