diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/CheckpointFactory.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/CheckpointFactory.java index 27205738..fe51584c 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/CheckpointFactory.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/CheckpointFactory.java @@ -15,15 +15,13 @@ package software.amazon.kinesis.checkpoint; -import software.amazon.kinesis.leases.LeaseManager; -import software.amazon.kinesis.leases.KinesisClientLease; import software.amazon.kinesis.leases.LeaseCoordinator; +import software.amazon.kinesis.leases.LeaseRefresher; import software.amazon.kinesis.processor.Checkpointer; /** * */ public interface CheckpointFactory { - Checkpointer createCheckpointer(LeaseCoordinator leaseCoordinator, - LeaseManager leaseManager); + Checkpointer createCheckpointer(LeaseCoordinator leaseCoordinator, LeaseRefresher leaseRefresher); } diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/DynamoDBCheckpointFactory.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/DynamoDBCheckpointFactory.java index 4a05d141..1a98c194 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/DynamoDBCheckpointFactory.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/DynamoDBCheckpointFactory.java @@ -17,9 +17,8 @@ package software.amazon.kinesis.checkpoint; import lombok.Data; import lombok.NonNull; -import software.amazon.kinesis.leases.LeaseManager; -import software.amazon.kinesis.leases.KinesisClientLease; import software.amazon.kinesis.leases.LeaseCoordinator; +import software.amazon.kinesis.leases.LeaseRefresher; import software.amazon.kinesis.metrics.IMetricsFactory; import software.amazon.kinesis.processor.Checkpointer; @@ -32,9 +31,9 @@ public class DynamoDBCheckpointFactory implements CheckpointFactory { private final IMetricsFactory metricsFactory; @Override - public Checkpointer createCheckpointer(final LeaseCoordinator leaseLeaseCoordinator, - final LeaseManager leaseManager) { - return new DynamoDBCheckpointer(leaseLeaseCoordinator, leaseManager, metricsFactory); + public Checkpointer createCheckpointer(final LeaseCoordinator leaseLeaseCoordinator, + final LeaseRefresher leaseRefresher) { + return new DynamoDBCheckpointer(leaseLeaseCoordinator, leaseRefresher, metricsFactory); } } diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/DynamoDBCheckpointer.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/DynamoDBCheckpointer.java index 2c614a82..042bfebd 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/DynamoDBCheckpointer.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/DynamoDBCheckpointer.java @@ -28,9 +28,9 @@ import com.google.common.annotations.VisibleForTesting; import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import software.amazon.kinesis.leases.LeaseManager; -import software.amazon.kinesis.leases.KinesisClientLease; +import software.amazon.kinesis.leases.Lease; import software.amazon.kinesis.leases.LeaseCoordinator; +import software.amazon.kinesis.leases.LeaseRefresher; import software.amazon.kinesis.leases.exceptions.DependencyException; import software.amazon.kinesis.leases.exceptions.InvalidStateException; import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException; @@ -45,9 +45,9 @@ import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; @Slf4j public class DynamoDBCheckpointer implements Checkpointer { @NonNull - private final LeaseCoordinator leaseCoordinator; + private final LeaseCoordinator leaseCoordinator; @NonNull - private final LeaseManager leaseManager; + private final LeaseRefresher leaseRefresher; @NonNull private final IMetricsFactory metricsFactory; @@ -73,7 +73,7 @@ public class DynamoDBCheckpointer implements Checkpointer { @Override public ExtendedSequenceNumber getCheckpoint(final String shardId) throws KinesisClientLibException { try { - return leaseManager.getLease(shardId).getCheckpoint(); + return leaseRefresher.getLease(shardId).checkpoint(); } catch (DependencyException | InvalidStateException | ProvisionedThroughputException e) { String message = "Unable to fetch checkpoint for shardId " + shardId; log.error(message, e); @@ -84,8 +84,8 @@ public class DynamoDBCheckpointer implements Checkpointer { @Override public Checkpoint getCheckpointObject(final String shardId) throws KinesisClientLibException { try { - KinesisClientLease lease = leaseManager.getLease(shardId); - return new Checkpoint(lease.getCheckpoint(), lease.getPendingCheckpoint()); + Lease lease = leaseRefresher.getLease(shardId); + return new Checkpoint(lease.checkpoint(), lease.pendingCheckpoint()); } catch (DependencyException | InvalidStateException | ProvisionedThroughputException e) { String message = "Unable to fetch checkpoint for shardId " + shardId; log.error(message, e); @@ -117,30 +117,30 @@ public class DynamoDBCheckpointer implements Checkpointer { @VisibleForTesting public boolean setCheckpoint(String shardId, ExtendedSequenceNumber checkpoint, UUID concurrencyToken) throws DependencyException, InvalidStateException, ProvisionedThroughputException { - KinesisClientLease lease = leaseCoordinator.getCurrentlyHeldLease(shardId); + Lease lease = leaseCoordinator.getCurrentlyHeldLease(shardId); if (lease == null) { log.info("Worker {} could not update checkpoint for shard {} because it does not hold the lease", - leaseCoordinator.getWorkerIdentifier(), shardId); + leaseCoordinator.workerIdentifier(), shardId); return false; } - lease.setCheckpoint(checkpoint); - lease.setPendingCheckpoint(null); - lease.setOwnerSwitchesSinceCheckpoint(0L); + lease.checkpoint(checkpoint); + lease.pendingCheckpoint(null); + lease.ownerSwitchesSinceCheckpoint(0L); return leaseCoordinator.updateLease(lease, concurrencyToken); } boolean prepareCheckpoint(String shardId, ExtendedSequenceNumber pendingCheckpoint, UUID concurrencyToken) throws DependencyException, InvalidStateException, ProvisionedThroughputException { - KinesisClientLease lease = leaseCoordinator.getCurrentlyHeldLease(shardId); + Lease lease = leaseCoordinator.getCurrentlyHeldLease(shardId); if (lease == null) { log.info("Worker {} could not prepare checkpoint for shard {} because it does not hold the lease", - leaseCoordinator.getWorkerIdentifier(), shardId); + leaseCoordinator.workerIdentifier(), shardId); return false; } - lease.setPendingCheckpoint(Objects.requireNonNull(pendingCheckpoint, "pendingCheckpoint should not be null")); + lease.pendingCheckpoint(Objects.requireNonNull(pendingCheckpoint, "pendingCheckpoint should not be null")); return leaseCoordinator.updateLease(lease, concurrencyToken); } } diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/Scheduler.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/Scheduler.java index 4167cba2..77e4929d 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/Scheduler.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/Scheduler.java @@ -38,11 +38,12 @@ import lombok.NonNull; import lombok.experimental.Accessors; import lombok.extern.slf4j.Slf4j; import software.amazon.kinesis.checkpoint.CheckpointConfig; -import software.amazon.kinesis.leases.LeaseManager; -import software.amazon.kinesis.leases.KinesisClientLease; -import software.amazon.kinesis.leases.KinesisClientLibLeaseCoordinator; +import software.amazon.kinesis.leases.dynamodb.DynamoDBLeaseCoordinator; +import software.amazon.kinesis.leases.Lease; +import software.amazon.kinesis.leases.LeaseCoordinator; import software.amazon.kinesis.leases.LeaseManagementConfig; -import software.amazon.kinesis.leases.LeaseManagerProxy; +import software.amazon.kinesis.leases.LeaseRefresher; +import software.amazon.kinesis.leases.ShardDetector; import software.amazon.kinesis.leases.ShardInfo; import software.amazon.kinesis.leases.ShardPrioritization; import software.amazon.kinesis.leases.ShardSyncTask; @@ -60,9 +61,9 @@ import software.amazon.kinesis.metrics.MetricsCollectingTaskDecorator; import software.amazon.kinesis.metrics.MetricsConfig; import software.amazon.kinesis.metrics.MetricsLevel; import software.amazon.kinesis.processor.Checkpointer; -import software.amazon.kinesis.processor.ShutdownNotificationAware; import software.amazon.kinesis.processor.ProcessorConfig; import software.amazon.kinesis.processor.ProcessorFactory; +import software.amazon.kinesis.processor.ShutdownNotificationAware; import software.amazon.kinesis.retrieval.RetrievalConfig; /** @@ -91,7 +92,7 @@ public class Scheduler implements Runnable { private final long parentShardPollIntervalMillis; private final ExecutorService executorService; // private final GetRecordsRetrievalStrategy getRecordsRetrievalStrategy; - private final KinesisClientLibLeaseCoordinator leaseCoordinator; + private final LeaseCoordinator leaseCoordinator; private final ShardSyncTaskManager shardSyncTaskManager; private final ShardPrioritization shardPrioritization; private final boolean cleanupLeasesUponShardCompletion; @@ -108,8 +109,8 @@ public class Scheduler implements Runnable { private final String streamName; private final long listShardsBackoffTimeMillis; private final int maxListShardsRetryAttempts; - private final LeaseManager leaseManager; - private final LeaseManagerProxy leaseManagerProxy; + private final LeaseRefresher leaseRefresher; + private final ShardDetector shardDetector; private final boolean ignoreUnexpetedChildShards; // Holds consumers for shards the worker is currently tracking. Key is shard @@ -143,15 +144,14 @@ public class Scheduler implements Runnable { this.retrievalConfig = retrievalConfig; this.applicationName = this.coordinatorConfig.applicationName(); - this.leaseCoordinator = - this.leaseManagementConfig.leaseManagementFactory().createKinesisClientLibLeaseCoordinator(); - this.leaseManager = this.leaseCoordinator.leaseManager(); + this.leaseCoordinator = this.leaseManagementConfig.leaseManagementFactory().createLeaseCoordinator(); + this.leaseRefresher = this.leaseCoordinator.leaseRefresher(); // // TODO: Figure out what to do with lease manage <=> checkpoint relationship // this.checkpoint = this.checkpointConfig.checkpointFactory().createCheckpointer(this.leaseCoordinator, - this.leaseManager); + this.leaseRefresher); this.idleTimeInMilliseconds = this.retrievalConfig.idleTimeBetweenReadsInMillis(); this.parentShardPollIntervalMillis = this.coordinatorConfig.parentShardPollIntervalMillis(); @@ -175,7 +175,7 @@ public class Scheduler implements Runnable { this.streamName = this.retrievalConfig.streamName(); this.listShardsBackoffTimeMillis = this.retrievalConfig.listShardsBackoffTimeInMillis(); this.maxListShardsRetryAttempts = this.retrievalConfig.maxListShardsRetryAttempts(); - this.leaseManagerProxy = this.shardSyncTaskManager.leaseManagerProxy(); + this.shardDetector = this.shardSyncTaskManager.shardDetector(); this.ignoreUnexpetedChildShards = this.leaseManagementConfig.ignoreUnexpectedChildShards(); } @@ -217,9 +217,9 @@ public class Scheduler implements Runnable { TaskResult result = null; if (!skipShardSyncAtWorkerInitializationIfLeasesExist - || leaseManager.isLeaseTableEmpty()) { + || leaseRefresher.isLeaseTableEmpty()) { log.info("Syncing Kinesis shard info"); - ShardSyncTask shardSyncTask = new ShardSyncTask(leaseManagerProxy, leaseManager, initialPosition, + ShardSyncTask shardSyncTask = new ShardSyncTask(shardDetector, leaseRefresher, initialPosition, cleanupLeasesUponShardCompletion, ignoreUnexpetedChildShards, 0L); result = new MetricsCollectingTaskDecorator(shardSyncTask, metricsFactory).call(); } else { @@ -398,7 +398,7 @@ public class Scheduler implements Runnable { // leaseCoordinator.stopLeaseTaker(); - Collection leases = leaseCoordinator.getAssignments(); + Collection leases = leaseCoordinator.getAssignments(); if (leases == null || leases.isEmpty()) { // // If there are no leases notification is already completed, but we still need to shutdown the worker. @@ -408,10 +408,10 @@ public class Scheduler implements Runnable { } CountDownLatch shutdownCompleteLatch = new CountDownLatch(leases.size()); CountDownLatch notificationCompleteLatch = new CountDownLatch(leases.size()); - for (KinesisClientLease lease : leases) { + for (Lease lease : leases) { ShutdownNotification shutdownNotification = new ShardConsumerShutdownNotification(leaseCoordinator, lease, notificationCompleteLatch, shutdownCompleteLatch); - ShardInfo shardInfo = KinesisClientLibLeaseCoordinator.convertLeaseToAssignment(lease); + ShardInfo shardInfo = DynamoDBLeaseCoordinator.convertLeaseToAssignment(lease); ShardConsumer consumer = shardInfoShardConsumerMap.get(shardInfo); if (consumer != null) { consumer.notifyShutdownRequested(shutdownNotification); @@ -531,7 +531,7 @@ public class Scheduler implements Runnable { @NonNull final ProcessorFactory processorFactory) { return new ShardConsumer(shardInfo, streamName, - leaseManager, + leaseRefresher, executorService, retrievalConfig.retrievalFactory().createGetRecordsCache(shardInfo, metricsFactory), processorFactory.createRecordProcessor(), @@ -551,7 +551,7 @@ public class Scheduler implements Runnable { initialPosition, cleanupLeasesUponShardCompletion, ignoreUnexpetedChildShards, - leaseManagerProxy, + shardDetector, metricsFactory); } diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/DynamoDBLeaseSerializer.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/DynamoDBLeaseSerializer.java deleted file mode 100644 index f832ab82..00000000 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/DynamoDBLeaseSerializer.java +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/asl/ - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ -package software.amazon.kinesis.leases; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import com.amazonaws.services.dynamodbv2.model.AttributeAction; -import com.amazonaws.services.dynamodbv2.model.AttributeDefinition; -import com.amazonaws.services.dynamodbv2.model.AttributeValue; -import com.amazonaws.services.dynamodbv2.model.AttributeValueUpdate; -import com.amazonaws.services.dynamodbv2.model.ExpectedAttributeValue; -import com.amazonaws.services.dynamodbv2.model.KeySchemaElement; -import com.amazonaws.services.dynamodbv2.model.KeyType; -import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType; - -/** - * An implementation of ILeaseSerializer for basic Lease objects. Can also instantiate subclasses of Lease so that - * LeaseSerializer can be decorated by other classes if you need to add fields to leases. - */ -public class DynamoDBLeaseSerializer implements LeaseSerializer { - - public final String LEASE_KEY_KEY = "leaseKey"; - public final String LEASE_OWNER_KEY = "leaseOwner"; - public final String LEASE_COUNTER_KEY = "leaseCounter"; - public final Class clazz; - - public DynamoDBLeaseSerializer() { - this.clazz = Lease.class; - } - - public DynamoDBLeaseSerializer(Class clazz) { - this.clazz = clazz; - } - - @Override - public Map toDynamoRecord(Lease lease) { - Map result = new HashMap(); - - result.put(LEASE_KEY_KEY, DynamoUtils.createAttributeValue(lease.getLeaseKey())); - result.put(LEASE_COUNTER_KEY, DynamoUtils.createAttributeValue(lease.getLeaseCounter())); - - if (lease.getLeaseOwner() != null) { - result.put(LEASE_OWNER_KEY, DynamoUtils.createAttributeValue(lease.getLeaseOwner())); - } - - return result; - } - - @Override - public Lease fromDynamoRecord(Map dynamoRecord) { - Lease result; - try { - result = clazz.newInstance(); - } catch (InstantiationException e) { - throw new RuntimeException(e); - } catch (IllegalAccessException e) { - throw new RuntimeException(e); - } - - result.setLeaseKey(DynamoUtils.safeGetString(dynamoRecord, LEASE_KEY_KEY)); - result.setLeaseOwner(DynamoUtils.safeGetString(dynamoRecord, LEASE_OWNER_KEY)); - result.setLeaseCounter(DynamoUtils.safeGetLong(dynamoRecord, LEASE_COUNTER_KEY)); - - return result; - } - - @Override - public Map getDynamoHashKey(String leaseKey) { - Map result = new HashMap(); - - result.put(LEASE_KEY_KEY, DynamoUtils.createAttributeValue(leaseKey)); - - return result; - } - - @Override - public Map getDynamoHashKey(Lease lease) { - return getDynamoHashKey(lease.getLeaseKey()); - } - - @Override - public Map getDynamoLeaseCounterExpectation(Lease lease) { - return getDynamoLeaseCounterExpectation(lease.getLeaseCounter()); - } - - public Map getDynamoLeaseCounterExpectation(Long leaseCounter) { - Map result = new HashMap(); - - ExpectedAttributeValue eav = new ExpectedAttributeValue(DynamoUtils.createAttributeValue(leaseCounter)); - result.put(LEASE_COUNTER_KEY, eav); - - return result; - } - - @Override - public Map getDynamoLeaseOwnerExpectation(Lease lease) { - Map result = new HashMap(); - - ExpectedAttributeValue eav = null; - - if (lease.getLeaseOwner() == null) { - eav = new ExpectedAttributeValue(false); - } else { - eav = new ExpectedAttributeValue(DynamoUtils.createAttributeValue(lease.getLeaseOwner())); - } - - result.put(LEASE_OWNER_KEY, eav); - - return result; - } - - @Override - public Map getDynamoNonexistantExpectation() { - Map result = new HashMap(); - - ExpectedAttributeValue expectedAV = new ExpectedAttributeValue(false); - result.put(LEASE_KEY_KEY, expectedAV); - - return result; - } - - @Override - public Map getDynamoLeaseCounterUpdate(Lease lease) { - return getDynamoLeaseCounterUpdate(lease.getLeaseCounter()); - } - - public Map getDynamoLeaseCounterUpdate(Long leaseCounter) { - Map result = new HashMap(); - - AttributeValueUpdate avu = - new AttributeValueUpdate(DynamoUtils.createAttributeValue(leaseCounter + 1), AttributeAction.PUT); - result.put(LEASE_COUNTER_KEY, avu); - - return result; - } - - @Override - public Map getDynamoTakeLeaseUpdate(Lease lease, String owner) { - Map result = new HashMap(); - - result.put(LEASE_OWNER_KEY, new AttributeValueUpdate(DynamoUtils.createAttributeValue(owner), - AttributeAction.PUT)); - - return result; - } - - @Override - public Map getDynamoEvictLeaseUpdate(Lease lease) { - Map result = new HashMap(); - - result.put(LEASE_OWNER_KEY, new AttributeValueUpdate(null, AttributeAction.DELETE)); - - return result; - } - - @Override - public Map getDynamoUpdateLeaseUpdate(Lease lease) { - // There is no application-specific data in Lease - just return a map that increments the counter. - return new HashMap(); - } - - @Override - public Collection getKeySchema() { - List keySchema = new ArrayList(); - keySchema.add(new KeySchemaElement().withAttributeName(LEASE_KEY_KEY).withKeyType(KeyType.HASH)); - - return keySchema; - } - - @Override - public Collection getAttributeDefinitions() { - List definitions = new ArrayList(); - definitions.add(new AttributeDefinition().withAttributeName(LEASE_KEY_KEY) - .withAttributeType(ScalarAttributeType.S)); - - return definitions; - } -} diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/KinesisClientDynamoDBLeaseManager.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/KinesisClientDynamoDBLeaseManager.java deleted file mode 100644 index 98c1a04d..00000000 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/KinesisClientDynamoDBLeaseManager.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/asl/ - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ -package software.amazon.kinesis.leases; - -import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; -import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; -import software.amazon.kinesis.leases.exceptions.DependencyException; -import software.amazon.kinesis.leases.exceptions.InvalidStateException; -import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException; - -/** - * An implementation of LeaseManager for the KinesisClientLibrary - takeLease updates the ownerSwitchesSinceCheckpoint field. - */ -public class KinesisClientDynamoDBLeaseManager extends DynamoDBLeaseManager implements KinesisClientLeaseManager { - /** - * Constructor. - * - * @param table Leases table - * @param dynamoDBClient DynamoDB client to use - */ - public KinesisClientDynamoDBLeaseManager(String table, AmazonDynamoDB dynamoDBClient) { - this(table, dynamoDBClient, false); - } - - /** - * Constructor for integration tests - see comment on superclass for documentation on setting the consistentReads - * flag. - * - * @param table leases table - * @param dynamoDBClient DynamoDB client to use - * @param consistentReads true if we want consistent reads for testing purposes. - */ - public KinesisClientDynamoDBLeaseManager(String table, AmazonDynamoDB dynamoDBClient, boolean consistentReads) { - super(table, dynamoDBClient, new KinesisClientLeaseSerializer(), consistentReads); - } - - /** - * {@inheritDoc} - */ - @Override - public boolean takeLease(KinesisClientLease lease, String newOwner) - throws DependencyException, InvalidStateException, ProvisionedThroughputException { - String oldOwner = lease.getLeaseOwner(); - - boolean result = super.takeLease(lease, newOwner); - - if (oldOwner != null && !oldOwner.equals(newOwner)) { - lease.setOwnerSwitchesSinceCheckpoint(lease.getOwnerSwitchesSinceCheckpoint() + 1); - } - - return result; - } - - /** - * {@inheritDoc} - */ - @Override - public ExtendedSequenceNumber getCheckpoint(String shardId) - throws ProvisionedThroughputException, InvalidStateException, DependencyException { - ExtendedSequenceNumber checkpoint = null; - KinesisClientLease lease = getLease(shardId); - if (lease != null) { - checkpoint = lease.getCheckpoint(); - } - return checkpoint; - } -} diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/KinesisClientLease.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/KinesisClientLease.java deleted file mode 100644 index 585ce9e8..00000000 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/KinesisClientLease.java +++ /dev/null @@ -1,207 +0,0 @@ -/* - * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/asl/ - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ -package software.amazon.kinesis.leases; - -import java.util.Collection; -import java.util.HashSet; -import java.util.Set; -import java.util.UUID; - -import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; - -/** - * A Lease subclass containing KinesisClientLibrary related fields for checkpoints. - */ -public class KinesisClientLease extends Lease { - - private ExtendedSequenceNumber checkpoint; - private ExtendedSequenceNumber pendingCheckpoint; - private Long ownerSwitchesSinceCheckpoint = 0L; - private Set parentShardIds = new HashSet(); - - public KinesisClientLease() { - - } - - public KinesisClientLease(KinesisClientLease other) { - super(other); - this.checkpoint = other.getCheckpoint(); - this.pendingCheckpoint = other.getPendingCheckpoint(); - this.ownerSwitchesSinceCheckpoint = other.getOwnerSwitchesSinceCheckpoint(); - this.parentShardIds.addAll(other.getParentShardIds()); - } - - KinesisClientLease(String leaseKey, String leaseOwner, Long leaseCounter, UUID concurrencyToken, - Long lastCounterIncrementNanos, ExtendedSequenceNumber checkpoint, ExtendedSequenceNumber pendingCheckpoint, - Long ownerSwitchesSinceCheckpoint, Set parentShardIds) { - super(leaseKey, leaseOwner, leaseCounter, concurrencyToken, lastCounterIncrementNanos); - - this.checkpoint = checkpoint; - this.pendingCheckpoint = pendingCheckpoint; - this.ownerSwitchesSinceCheckpoint = ownerSwitchesSinceCheckpoint; - this.parentShardIds.addAll(parentShardIds); - } - - /** - * {@inheritDoc} - */ - @Override - public void update(T other) { - super.update(other); - if (!(other instanceof KinesisClientLease)) { - throw new IllegalArgumentException("Must pass KinesisClientLease object to KinesisClientLease.update(Lease)"); - } - KinesisClientLease casted = (KinesisClientLease) other; - - setOwnerSwitchesSinceCheckpoint(casted.ownerSwitchesSinceCheckpoint); - setCheckpoint(casted.checkpoint); - setPendingCheckpoint(casted.pendingCheckpoint); - setParentShardIds(casted.parentShardIds); - } - - /** - * @return most recently application-supplied checkpoint value. During fail over, the new worker will pick up after - * the old worker's last checkpoint. - */ - public ExtendedSequenceNumber getCheckpoint() { - return checkpoint; - } - - /** - * @return pending checkpoint, possibly null. - */ - public ExtendedSequenceNumber getPendingCheckpoint() { - return pendingCheckpoint; - } - - /** - * @return count of distinct lease holders between checkpoints. - */ - public Long getOwnerSwitchesSinceCheckpoint() { - return ownerSwitchesSinceCheckpoint; - } - - /** - * @return shardIds that parent this lease. Used for resharding. - */ - public Set getParentShardIds() { - return new HashSet(parentShardIds); - } - - /** - * Sets checkpoint. - * - * @param checkpoint may not be null - */ - public void setCheckpoint(ExtendedSequenceNumber checkpoint) { - verifyNotNull(checkpoint, "Checkpoint should not be null"); - - this.checkpoint = checkpoint; - } - - /** - * Sets pending checkpoint. - * - * @param pendingCheckpoint can be null - */ - public void setPendingCheckpoint(ExtendedSequenceNumber pendingCheckpoint) { - this.pendingCheckpoint = pendingCheckpoint; - } - - /** - * Sets ownerSwitchesSinceCheckpoint. - * - * @param ownerSwitchesSinceCheckpoint may not be null - */ - public void setOwnerSwitchesSinceCheckpoint(Long ownerSwitchesSinceCheckpoint) { - verifyNotNull(ownerSwitchesSinceCheckpoint, "ownerSwitchesSinceCheckpoint should not be null"); - - this.ownerSwitchesSinceCheckpoint = ownerSwitchesSinceCheckpoint; - } - - /** - * Sets parentShardIds. - * - * @param parentShardIds may not be null - */ - public void setParentShardIds(Collection parentShardIds) { - verifyNotNull(parentShardIds, "parentShardIds should not be null"); - - this.parentShardIds.clear(); - this.parentShardIds.addAll(parentShardIds); - } - - private void verifyNotNull(Object object, String message) { - if (object == null) { - throw new IllegalArgumentException(message); - } - } - - @Override - public int hashCode() { - final int prime = 31; - int result = super.hashCode(); - result = prime * result + ((checkpoint == null) ? 0 : checkpoint.hashCode()); - result = pendingCheckpoint == null ? result : prime * result + pendingCheckpoint.hashCode(); - result = - prime * result + ((ownerSwitchesSinceCheckpoint == null) ? 0 : ownerSwitchesSinceCheckpoint.hashCode()); - result = prime * result + ((parentShardIds == null) ? 0 : parentShardIds.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (!super.equals(obj)) - return false; - if (getClass() != obj.getClass()) - return false; - KinesisClientLease other = (KinesisClientLease) obj; - if (checkpoint == null) { - if (other.checkpoint != null) - return false; - } else if (!checkpoint.equals(other.checkpoint)) - return false; - if (pendingCheckpoint == null) { - if (other.pendingCheckpoint != null) - return false; - } else if (!pendingCheckpoint.equals(other.pendingCheckpoint)) - return false; - if (ownerSwitchesSinceCheckpoint == null) { - if (other.ownerSwitchesSinceCheckpoint != null) - return false; - } else if (!ownerSwitchesSinceCheckpoint.equals(other.ownerSwitchesSinceCheckpoint)) - return false; - if (parentShardIds == null) { - if (other.parentShardIds != null) - return false; - } else if (!parentShardIds.equals(other.parentShardIds)) - return false; - return true; - } - - /** - * Returns a deep copy of this object. Type-unsafe - there aren't good mechanisms for copy-constructing generics. - * - * @return A deep copy of this object. - */ - @Override - @SuppressWarnings("unchecked") - public T copy() { - return (T) new KinesisClientLease(this); - } - -} diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/KinesisClientLeaseManager.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/KinesisClientLeaseManager.java deleted file mode 100644 index 9426ffd3..00000000 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/KinesisClientLeaseManager.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/asl/ - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ -package software.amazon.kinesis.leases; - -import software.amazon.kinesis.leases.exceptions.DependencyException; -import software.amazon.kinesis.leases.exceptions.InvalidStateException; -import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException; -import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; - -/** - * A decoration of ILeaseManager that adds methods to get/update checkpoints. - */ -public interface KinesisClientLeaseManager extends LeaseManager { - - /** - * Gets the current checkpoint of the shard. This is useful in the resharding use case - * where we will wait for the parent shard to complete before starting on the records from a child shard. - * - * @param shardId Checkpoint of this shard will be returned - * @return Checkpoint of this shard, or null if the shard record doesn't exist. - * - * @throws ProvisionedThroughputException if DynamoDB update fails due to lack of capacity - * @throws InvalidStateException if lease table does not exist - * @throws DependencyException if DynamoDB update fails in an unexpected way - */ - ExtendedSequenceNumber getCheckpoint(String shardId) throws ProvisionedThroughputException, InvalidStateException, DependencyException; - -} diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/KinesisClientLeaseSerializer.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/KinesisClientLeaseSerializer.java deleted file mode 100644 index 87eaa86a..00000000 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/KinesisClientLeaseSerializer.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/asl/ - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ -package software.amazon.kinesis.leases; - -import java.util.Collection; -import java.util.Map; - -import com.amazonaws.services.dynamodbv2.model.AttributeAction; -import com.amazonaws.services.dynamodbv2.model.AttributeDefinition; -import com.amazonaws.services.dynamodbv2.model.AttributeValue; -import com.amazonaws.services.dynamodbv2.model.AttributeValueUpdate; -import com.amazonaws.services.dynamodbv2.model.ExpectedAttributeValue; -import com.amazonaws.services.dynamodbv2.model.KeySchemaElement; -import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; -import com.google.common.base.Strings; - -/** - * An implementation of ILeaseSerializer for KinesisClientLease objects. - */ -public class KinesisClientLeaseSerializer implements LeaseSerializer { - - private static final String OWNER_SWITCHES_KEY = "ownerSwitchesSinceCheckpoint"; - private static final String CHECKPOINT_SEQUENCE_NUMBER_KEY = "checkpoint"; - private static final String CHECKPOINT_SUBSEQUENCE_NUMBER_KEY = "checkpointSubSequenceNumber"; - private static final String PENDING_CHECKPOINT_SEQUENCE_KEY = "pendingCheckpoint"; - private static final String PENDING_CHECKPOINT_SUBSEQUENCE_KEY = "pendingCheckpointSubSequenceNumber"; - public final String PARENT_SHARD_ID_KEY = "parentShardId"; - - private final DynamoDBLeaseSerializer baseSerializer = new DynamoDBLeaseSerializer(KinesisClientLease.class); - - @Override - public Map toDynamoRecord(KinesisClientLease lease) { - Map result = baseSerializer.toDynamoRecord(lease); - - result.put(OWNER_SWITCHES_KEY, DynamoUtils.createAttributeValue(lease.getOwnerSwitchesSinceCheckpoint())); - result.put(CHECKPOINT_SEQUENCE_NUMBER_KEY, DynamoUtils.createAttributeValue(lease.getCheckpoint().getSequenceNumber())); - result.put(CHECKPOINT_SUBSEQUENCE_NUMBER_KEY, DynamoUtils.createAttributeValue(lease.getCheckpoint().getSubSequenceNumber())); - if (lease.getParentShardIds() != null && !lease.getParentShardIds().isEmpty()) { - result.put(PARENT_SHARD_ID_KEY, DynamoUtils.createAttributeValue(lease.getParentShardIds())); - } - - if (lease.getPendingCheckpoint() != null && !lease.getPendingCheckpoint().getSequenceNumber().isEmpty()) { - result.put(PENDING_CHECKPOINT_SEQUENCE_KEY, DynamoUtils.createAttributeValue(lease.getPendingCheckpoint().getSequenceNumber())); - result.put(PENDING_CHECKPOINT_SUBSEQUENCE_KEY, DynamoUtils.createAttributeValue(lease.getPendingCheckpoint().getSubSequenceNumber())); - } - - return result; - } - - @Override - public KinesisClientLease fromDynamoRecord(Map dynamoRecord) { - KinesisClientLease result = (KinesisClientLease) baseSerializer.fromDynamoRecord(dynamoRecord); - - result.setOwnerSwitchesSinceCheckpoint(DynamoUtils.safeGetLong(dynamoRecord, OWNER_SWITCHES_KEY)); - result.setCheckpoint( - new ExtendedSequenceNumber( - DynamoUtils.safeGetString(dynamoRecord, CHECKPOINT_SEQUENCE_NUMBER_KEY), - DynamoUtils.safeGetLong(dynamoRecord, CHECKPOINT_SUBSEQUENCE_NUMBER_KEY)) - ); - result.setParentShardIds(DynamoUtils.safeGetSS(dynamoRecord, PARENT_SHARD_ID_KEY)); - - if (!Strings.isNullOrEmpty(DynamoUtils.safeGetString(dynamoRecord, PENDING_CHECKPOINT_SEQUENCE_KEY))) { - result.setPendingCheckpoint( - new ExtendedSequenceNumber( - DynamoUtils.safeGetString(dynamoRecord, PENDING_CHECKPOINT_SEQUENCE_KEY), - DynamoUtils.safeGetLong(dynamoRecord, PENDING_CHECKPOINT_SUBSEQUENCE_KEY)) - ); - } - - return result; - } - - @Override - public Map getDynamoHashKey(KinesisClientLease lease) { - return baseSerializer.getDynamoHashKey(lease); - } - - @Override - public Map getDynamoHashKey(String shardId) { - return baseSerializer.getDynamoHashKey(shardId); - } - - @Override - public Map getDynamoLeaseCounterExpectation(KinesisClientLease lease) { - return baseSerializer.getDynamoLeaseCounterExpectation(lease); - } - - @Override - public Map getDynamoLeaseOwnerExpectation(KinesisClientLease lease) { - return baseSerializer.getDynamoLeaseOwnerExpectation(lease); - } - - @Override - public Map getDynamoNonexistantExpectation() { - return baseSerializer.getDynamoNonexistantExpectation(); - } - - @Override - public Map getDynamoLeaseCounterUpdate(KinesisClientLease lease) { - return baseSerializer.getDynamoLeaseCounterUpdate(lease); - } - - @Override - public Map getDynamoTakeLeaseUpdate(KinesisClientLease lease, String newOwner) { - Map result = baseSerializer.getDynamoTakeLeaseUpdate(lease, newOwner); - - String oldOwner = lease.getLeaseOwner(); - if (oldOwner != null && !oldOwner.equals(newOwner)) { - result.put(OWNER_SWITCHES_KEY, new AttributeValueUpdate(DynamoUtils.createAttributeValue(1L), - AttributeAction.ADD)); - } - - return result; - } - - @Override - public Map getDynamoEvictLeaseUpdate(KinesisClientLease lease) { - return baseSerializer.getDynamoEvictLeaseUpdate(lease); - } - - @Override - public Map getDynamoUpdateLeaseUpdate(KinesisClientLease lease) { - Map result = baseSerializer.getDynamoUpdateLeaseUpdate(lease); - - result.put(CHECKPOINT_SEQUENCE_NUMBER_KEY, new AttributeValueUpdate(DynamoUtils.createAttributeValue(lease.getCheckpoint().getSequenceNumber()), - AttributeAction.PUT)); - result.put(CHECKPOINT_SUBSEQUENCE_NUMBER_KEY, new AttributeValueUpdate(DynamoUtils.createAttributeValue(lease.getCheckpoint().getSubSequenceNumber()), - AttributeAction.PUT)); - result.put(OWNER_SWITCHES_KEY, - new AttributeValueUpdate(DynamoUtils.createAttributeValue(lease.getOwnerSwitchesSinceCheckpoint()), - AttributeAction.PUT)); - - if (lease.getPendingCheckpoint() != null && !lease.getPendingCheckpoint().getSequenceNumber().isEmpty()) { - result.put(PENDING_CHECKPOINT_SEQUENCE_KEY, new AttributeValueUpdate(DynamoUtils.createAttributeValue(lease.getPendingCheckpoint().getSequenceNumber()), AttributeAction.PUT)); - result.put(PENDING_CHECKPOINT_SUBSEQUENCE_KEY, new AttributeValueUpdate(DynamoUtils.createAttributeValue(lease.getPendingCheckpoint().getSubSequenceNumber()), AttributeAction.PUT)); - } else { - result.put(PENDING_CHECKPOINT_SEQUENCE_KEY, new AttributeValueUpdate().withAction(AttributeAction.DELETE)); - result.put(PENDING_CHECKPOINT_SUBSEQUENCE_KEY, new AttributeValueUpdate().withAction(AttributeAction.DELETE)); - } - - return result; - } - - @Override - public Collection getKeySchema() { - return baseSerializer.getKeySchema(); - } - - @Override - public Collection getAttributeDefinitions() { - return baseSerializer.getAttributeDefinitions(); - } - -} diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/KinesisClientLibLeaseCoordinator.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/KinesisClientLibLeaseCoordinator.java deleted file mode 100644 index 031f01f0..00000000 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/KinesisClientLibLeaseCoordinator.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/asl/ - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ -package software.amazon.kinesis.leases; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Set; - -import lombok.Getter; -import lombok.experimental.Accessors; -import lombok.extern.slf4j.Slf4j; -import software.amazon.kinesis.leases.exceptions.DependencyException; -import software.amazon.kinesis.leases.exceptions.InvalidStateException; -import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException; -import software.amazon.kinesis.metrics.IMetricsFactory; - -/** - * This class is used to coordinate/manage leases owned by this worker process and to get/set checkpoints. - */ -@Slf4j -public class KinesisClientLibLeaseCoordinator extends LeaseCoordinator { - private static final long DEFAULT_INITIAL_LEASE_TABLE_READ_CAPACITY = 10L; - private static final long DEFAULT_INITIAL_LEASE_TABLE_WRITE_CAPACITY = 10L; - - @Getter - @Accessors(fluent = true) - private final LeaseManager leaseManager; - - private long initialLeaseTableReadCapacity = DEFAULT_INITIAL_LEASE_TABLE_READ_CAPACITY; - private long initialLeaseTableWriteCapacity = DEFAULT_INITIAL_LEASE_TABLE_WRITE_CAPACITY; - - public KinesisClientLibLeaseCoordinator(final LeaseManager leaseManager, - final String workerIdentifier, - final long leaseDurationMillis, - final long epsilonMillis, - final int maxLeasesForWorker, - final int maxLeasesToStealAtOneTime, - final int maxLeaseRenewerThreadCount, - final IMetricsFactory metricsFactory) { - super(leaseManager, workerIdentifier, leaseDurationMillis, epsilonMillis, maxLeasesForWorker, - maxLeasesToStealAtOneTime, maxLeaseRenewerThreadCount, metricsFactory); - this.leaseManager = leaseManager; - } - - /** - * @param readCapacity The DynamoDB table used for tracking leases will be provisioned with the specified initial - * read capacity - * @return KinesisClientLibLeaseCoordinator - */ - public KinesisClientLibLeaseCoordinator initialLeaseTableReadCapacity(long readCapacity) { - if (readCapacity <= 0) { - throw new IllegalArgumentException("readCapacity should be >= 1"); - } - this.initialLeaseTableReadCapacity = readCapacity; - return this; - } - - /** - * @param writeCapacity The DynamoDB table used for tracking leases will be provisioned with the specified initial - * write capacity - * @return KinesisClientLibLeaseCoordinator - */ - public KinesisClientLibLeaseCoordinator initialLeaseTableWriteCapacity(long writeCapacity) { - if (writeCapacity <= 0) { - throw new IllegalArgumentException("writeCapacity should be >= 1"); - } - this.initialLeaseTableWriteCapacity = writeCapacity; - return this; - } - - /** - * @return Current shard/lease assignments - */ - public List getCurrentAssignments() { - Collection leases = getAssignments(); - return convertLeasesToAssignments(leases); - - } - - public static List convertLeasesToAssignments(Collection leases) { - if (leases == null || leases.isEmpty()) { - return Collections.emptyList(); - } - List assignments = new ArrayList<>(leases.size()); - for (KinesisClientLease lease : leases) { - assignments.add(convertLeaseToAssignment(lease)); - } - - return assignments; - } - - public static ShardInfo convertLeaseToAssignment(KinesisClientLease lease) { - Set parentShardIds = lease.getParentShardIds(); - return new ShardInfo(lease.getLeaseKey(), lease.getConcurrencyToken().toString(), parentShardIds, - lease.getCheckpoint()); - } - - /** - * Initialize the lease coordinator (create the lease table if needed). - * @throws DependencyException - * @throws ProvisionedThroughputException - */ - public void initialize() throws ProvisionedThroughputException, DependencyException, IllegalStateException { - final boolean newTableCreated = - leaseManager.createLeaseTableIfNotExists(initialLeaseTableReadCapacity, initialLeaseTableWriteCapacity); - if (newTableCreated) { - log.info("Created new lease table for coordinator with initial read capacity of {} and write capacity of {}.", - initialLeaseTableReadCapacity, initialLeaseTableWriteCapacity); - } - // Need to wait for table in active state. - final long secondsBetweenPolls = 10L; - final long timeoutSeconds = 600L; - final boolean isTableActive = leaseManager.waitUntilLeaseTableExists(secondsBetweenPolls, timeoutSeconds); - if (!isTableActive) { - throw new DependencyException(new IllegalStateException("Creating table timeout")); - } - } - - /** - * Package access for testing. - * - * @throws DependencyException - * @throws InvalidStateException - */ - public void runLeaseTaker() throws DependencyException, InvalidStateException { - super.runTaker(); - } - - /** - * Package access for testing. - * - * @throws DependencyException - * @throws InvalidStateException - */ - public void runLeaseRenewer() throws DependencyException, InvalidStateException { - super.runRenewer(); - } - -} diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/KinesisLeaseManagerProxy.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/KinesisShardDetector.java similarity index 98% rename from amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/KinesisLeaseManagerProxy.java rename to amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/KinesisShardDetector.java index 7dbeb46c..b51adbf9 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/KinesisLeaseManagerProxy.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/KinesisShardDetector.java @@ -36,7 +36,7 @@ import lombok.extern.slf4j.Slf4j; */ @RequiredArgsConstructor @Slf4j -public class KinesisLeaseManagerProxy implements LeaseManagerProxy { +public class KinesisShardDetector implements ShardDetector { @NonNull private final AmazonKinesis amazonKinesis; @NonNull diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/Lease.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/Lease.java index 05f7e731..08d08ce1 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/Lease.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/Lease.java @@ -14,10 +14,20 @@ */ package software.amazon.kinesis.leases; +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; -import com.amazonaws.util.json.Jackson; +import com.amazonaws.util.CollectionUtils; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.NonNull; +import lombok.ToString; +import lombok.experimental.Accessors; +import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; /** * This class contains data pertaining to a Lease. Distributed systems may use leases to partition work across a @@ -26,6 +36,11 @@ import com.amazonaws.util.json.Jackson; * processing the corresponding unit of work, or until it fails. When the worker stops holding the lease, another worker will * take and hold the lease. */ +@NoArgsConstructor +@Getter +@Accessors(fluent = true) +@EqualsAndHashCode(exclude = {"concurrencyToken", "lastCounterIncrementNanos"}) +@ToString public class Lease { /* * See javadoc for System.nanoTime - summary: @@ -35,8 +50,17 @@ public class Lease { */ private static final long MAX_ABS_AGE_NANOS = TimeUnit.DAYS.toNanos(365); + /** + * @return leaseKey - identifies the unit of work associated with this lease. + */ private String leaseKey; + /** + * @return current owner of the lease, may be null. + */ private String leaseOwner; + /** + * @return leaseCounter is incremented periodically by the holder of the lease. Used for optimistic locking. + */ private Long leaseCounter = 0L; /* @@ -50,12 +74,20 @@ public class Lease { * deliberately not persisted in DynamoDB and excluded from hashCode and equals. */ private Long lastCounterIncrementNanos; - /** - * Constructor. + * @return most recently application-supplied checkpoint value. During fail over, the new worker will pick up after + * the old worker's last checkpoint. */ - public Lease() { - } + private ExtendedSequenceNumber checkpoint; + /** + * @return pending checkpoint, possibly null. + */ + private ExtendedSequenceNumber pendingCheckpoint; + /** + * @return count of distinct lease holders between checkpoints. + */ + private Long ownerSwitchesSinceCheckpoint = 0L; + private Set parentShardIds = new HashSet<>(); /** * Copy constructor, used by clone(). @@ -63,62 +95,46 @@ public class Lease { * @param lease lease to copy */ protected Lease(Lease lease) { - this(lease.getLeaseKey(), lease.getLeaseOwner(), lease.getLeaseCounter(), lease.getConcurrencyToken(), - lease.getLastCounterIncrementNanos()); + this(lease.leaseKey(), lease.leaseOwner(), lease.leaseCounter(), lease.concurrencyToken(), + lease.lastCounterIncrementNanos(), lease.checkpoint(), lease.pendingCheckpoint(), + lease.ownerSwitchesSinceCheckpoint(), lease.parentShardIds()); } - protected Lease(String leaseKey, String leaseOwner, Long leaseCounter, UUID concurrencyToken, - Long lastCounterIncrementNanos) { + public Lease(final String leaseKey, final String leaseOwner, final Long leaseCounter, + final UUID concurrencyToken, final Long lastCounterIncrementNanos, + final ExtendedSequenceNumber checkpoint, final ExtendedSequenceNumber pendingCheckpoint, + final Long ownerSwitchesSinceCheckpoint, final Set parentShardIds) { this.leaseKey = leaseKey; this.leaseOwner = leaseOwner; this.leaseCounter = leaseCounter; this.concurrencyToken = concurrencyToken; this.lastCounterIncrementNanos = lastCounterIncrementNanos; + this.checkpoint = checkpoint; + this.pendingCheckpoint = pendingCheckpoint; + this.ownerSwitchesSinceCheckpoint = ownerSwitchesSinceCheckpoint; + if (!CollectionUtils.isNullOrEmpty(parentShardIds)) { + this.parentShardIds.addAll(parentShardIds); + } + } + + /** + * @return shardIds that parent this lease. Used for resharding. + */ + public Set parentShardIds() { + return new HashSet<>(parentShardIds); } /** * Updates this Lease's mutable, application-specific fields based on the passed-in lease object. Does not update * fields that are internal to the leasing library (leaseKey, leaseOwner, leaseCounter). * - * @param other + * @param lease */ - public void update(T other) { - // The default implementation (no application-specific fields) has nothing to do. - } - - /** - * @return leaseKey - identifies the unit of work associated with this lease. - */ - public String getLeaseKey() { - return leaseKey; - } - - /** - * @return leaseCounter is incremented periodically by the holder of the lease. Used for optimistic locking. - */ - public Long getLeaseCounter() { - return leaseCounter; - } - - /** - * @return current owner of the lease, may be null. - */ - public String getLeaseOwner() { - return leaseOwner; - } - - /** - * @return concurrency token - */ - public UUID getConcurrencyToken() { - return concurrencyToken; - } - - /** - * @return last update in nanoseconds since the epoch - */ - public Long getLastCounterIncrementNanos() { - return lastCounterIncrementNanos; + public void update(final Lease lease) { + ownerSwitchesSinceCheckpoint(lease.ownerSwitchesSinceCheckpoint()); + checkpoint(lease.checkpoint); + pendingCheckpoint(lease.pendingCheckpoint); + parentShardIds(lease.parentShardIds); } /** @@ -145,7 +161,7 @@ public class Lease { * * @param lastCounterIncrementNanos last renewal in nanoseconds since the epoch */ - public void setLastCounterIncrementNanos(Long lastCounterIncrementNanos) { + public void lastCounterIncrementNanos(Long lastCounterIncrementNanos) { this.lastCounterIncrementNanos = lastCounterIncrementNanos; } @@ -154,8 +170,7 @@ public class Lease { * * @param concurrencyToken may not be null */ - public void setConcurrencyToken(UUID concurrencyToken) { - verifyNotNull(concurrencyToken, "concurencyToken cannot be null"); + public void concurrencyToken(@NonNull final UUID concurrencyToken) { this.concurrencyToken = concurrencyToken; } @@ -164,12 +179,10 @@ public class Lease { * * @param leaseKey may not be null. */ - public void setLeaseKey(String leaseKey) { + public void leaseKey(@NonNull final String leaseKey) { if (this.leaseKey != null) { throw new IllegalArgumentException("LeaseKey is immutable once set"); } - verifyNotNull(leaseKey, "LeaseKey cannot be set to null"); - this.leaseKey = leaseKey; } @@ -178,77 +191,62 @@ public class Lease { * * @param leaseCounter may not be null */ - public void setLeaseCounter(Long leaseCounter) { - verifyNotNull(leaseCounter, "leaseCounter must not be null"); - + public void leaseCounter(@NonNull final Long leaseCounter) { this.leaseCounter = leaseCounter; } + /** + * Sets checkpoint. + * + * @param checkpoint may not be null + */ + public void checkpoint(@NonNull final ExtendedSequenceNumber checkpoint) { + this.checkpoint = checkpoint; + } + + /** + * Sets pending checkpoint. + * + * @param pendingCheckpoint can be null + */ + public void pendingCheckpoint(ExtendedSequenceNumber pendingCheckpoint) { + this.pendingCheckpoint = pendingCheckpoint; + } + + /** + * Sets ownerSwitchesSinceCheckpoint. + * + * @param ownerSwitchesSinceCheckpoint may not be null + */ + public void ownerSwitchesSinceCheckpoint(@NonNull final Long ownerSwitchesSinceCheckpoint) { + this.ownerSwitchesSinceCheckpoint = ownerSwitchesSinceCheckpoint; + } + + /** + * Sets parentShardIds. + * + * @param parentShardIds may not be null + */ + public void parentShardIds(@NonNull final Collection parentShardIds) { + this.parentShardIds.clear(); + this.parentShardIds.addAll(parentShardIds); + } + /** * Sets leaseOwner. * * @param leaseOwner may be null. */ - public void setLeaseOwner(String leaseOwner) { + public void leaseOwner(String leaseOwner) { this.leaseOwner = leaseOwner; } - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((leaseCounter == null) ? 0 : leaseCounter.hashCode()); - result = prime * result + ((leaseOwner == null) ? 0 : leaseOwner.hashCode()); - result = prime * result + ((leaseKey == null) ? 0 : leaseKey.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - Lease other = (Lease) obj; - if (leaseCounter == null) { - if (other.leaseCounter != null) - return false; - } else if (!leaseCounter.equals(other.leaseCounter)) - return false; - if (leaseOwner == null) { - if (other.leaseOwner != null) - return false; - } else if (!leaseOwner.equals(other.leaseOwner)) - return false; - if (leaseKey == null) { - if (other.leaseKey != null) - return false; - } else if (!leaseKey.equals(other.leaseKey)) - return false; - return true; - } - - @Override - public String toString() { - return Jackson.toJsonPrettyString(this); - } - /** * Returns a deep copy of this object. Type-unsafe - there aren't good mechanisms for copy-constructing generics. * * @return A deep copy of this object. */ - @SuppressWarnings("unchecked") - public T copy() { - return (T) new Lease(this); + public Lease copy() { + return new Lease(this); } - - private void verifyNotNull(Object object, String message) { - if (object == null) { - throw new IllegalArgumentException(message); - } - } - } diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseCoordinator.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseCoordinator.java index 07c3394c..a206c7ee 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseCoordinator.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseCoordinator.java @@ -12,175 +12,28 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ + package software.amazon.kinesis.leases; import java.util.Collection; -import java.util.Map; +import java.util.List; import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedTransferQueue; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import com.google.common.util.concurrent.ThreadFactoryBuilder; - -import lombok.extern.slf4j.Slf4j; +import software.amazon.kinesis.leases.dynamodb.DynamoDBLeaseCoordinator; import software.amazon.kinesis.leases.exceptions.DependencyException; import software.amazon.kinesis.leases.exceptions.InvalidStateException; -import software.amazon.kinesis.leases.exceptions.LeasingException; import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException; -import software.amazon.kinesis.metrics.IMetricsFactory; -import software.amazon.kinesis.metrics.IMetricsScope; -import software.amazon.kinesis.metrics.LogMetricsFactory; -import software.amazon.kinesis.metrics.MetricsHelper; -import software.amazon.kinesis.metrics.MetricsLevel; /** - * LeaseCoordinator abstracts away LeaseTaker and LeaseRenewer from the application code that's using leasing. It owns - * the scheduling of the two previously mentioned components as well as informing LeaseRenewer when LeaseTaker takes new - * leases. * */ -@Slf4j -public class LeaseCoordinator { - - /* - * Name of the dimension used when setting worker identifier on IMetricsScopes. Exposed so that users of this class - * can easily create InterceptingMetricsFactories that rename this dimension to suit the destination metrics system. - */ - public static final String WORKER_IDENTIFIER_METRIC = "WorkerIdentifier"; - - // Time to wait for in-flight Runnables to finish when calling .stop(); - private static final long STOP_WAIT_TIME_MILLIS = 2000L; - - private static final int DEFAULT_MAX_LEASES_FOR_WORKER = Integer.MAX_VALUE; - private static final int DEFAULT_MAX_LEASES_TO_STEAL_AT_ONE_TIME = 1; - - private static final ThreadFactory LEASE_COORDINATOR_THREAD_FACTORY = new ThreadFactoryBuilder() - .setNameFormat("LeaseCoordinator-%04d").setDaemon(true).build(); - private static final ThreadFactory LEASE_RENEWAL_THREAD_FACTORY = new ThreadFactoryBuilder() - .setNameFormat("LeaseRenewer-%04d").setDaemon(true).build(); - - private final LeaseRenewer leaseRenewer; - private final LeaseTaker leaseTaker; - private final long renewerIntervalMillis; - private final long takerIntervalMillis; - - private final Object shutdownLock = new Object(); - - protected final IMetricsFactory metricsFactory; - - private ScheduledExecutorService leaseCoordinatorThreadPool; - private final ExecutorService leaseRenewalThreadpool; - private volatile boolean running = false; - private ScheduledFuture takerFuture; - +public interface LeaseCoordinator { /** - * Constructor. - * - * @param leaseManager LeaseManager instance to use - * @param workerIdentifier Identifies the worker (e.g. useful to track lease ownership) - * @param leaseDurationMillis Duration of a lease - * @param epsilonMillis Allow for some variance when calculating lease expirations + * Initialize the lease coordinator (create the lease table if needed). + * @throws DependencyException + * @throws ProvisionedThroughputException */ - public LeaseCoordinator(LeaseManager leaseManager, - String workerIdentifier, - long leaseDurationMillis, - long epsilonMillis) { - this(leaseManager, workerIdentifier, leaseDurationMillis, epsilonMillis, new LogMetricsFactory()); - } - - /** - * Constructor. - * - * @param leaseManager LeaseManager instance to use - * @param workerIdentifier Identifies the worker (e.g. useful to track lease ownership) - * @param leaseDurationMillis Duration of a lease - * @param epsilonMillis Allow for some variance when calculating lease expirations - * @param metricsFactory Used to publish metrics about lease operations - */ - public LeaseCoordinator(LeaseManager leaseManager, - String workerIdentifier, - long leaseDurationMillis, - long epsilonMillis, - IMetricsFactory metricsFactory) { - this(leaseManager, workerIdentifier, leaseDurationMillis, epsilonMillis, - DEFAULT_MAX_LEASES_FOR_WORKER, DEFAULT_MAX_LEASES_TO_STEAL_AT_ONE_TIME, - LeaseManagementConfig.DEFAULT_MAX_LEASE_RENEWAL_THREADS, metricsFactory); - } - - /** - * Constructor. - * - * @param leaseManager LeaseManager instance to use - * @param workerIdentifier Identifies the worker (e.g. useful to track lease ownership) - * @param leaseDurationMillis Duration of a lease - * @param epsilonMillis Allow for some variance when calculating lease expirations - * @param maxLeasesForWorker Max leases this Worker can handle at a time - * @param maxLeasesToStealAtOneTime Steal up to these many leases at a time (for load balancing) - * @param metricsFactory Used to publish metrics about lease operations - */ - public LeaseCoordinator(LeaseManager leaseManager, - String workerIdentifier, - long leaseDurationMillis, - long epsilonMillis, - int maxLeasesForWorker, - int maxLeasesToStealAtOneTime, - int maxLeaseRenewerThreadCount, - IMetricsFactory metricsFactory) { - this.leaseRenewalThreadpool = getLeaseRenewalExecutorService(maxLeaseRenewerThreadCount); - this.leaseTaker = new DynamoDBLeaseTaker(leaseManager, workerIdentifier, leaseDurationMillis) - .withMaxLeasesForWorker(maxLeasesForWorker) - .withMaxLeasesToStealAtOneTime(maxLeasesToStealAtOneTime); - this.leaseRenewer = new DynamoDBLeaseRenewer( - leaseManager, workerIdentifier, leaseDurationMillis, leaseRenewalThreadpool); - this.renewerIntervalMillis = leaseDurationMillis / 3 - epsilonMillis; - this.takerIntervalMillis = (leaseDurationMillis + epsilonMillis) * 2; - this.metricsFactory = metricsFactory; - - log.info("With failover time {} ms and epsilon {} ms, LeaseCoordinator will renew leases every {} ms, take" - + "leases every {} ms, process maximum of {} leases and steal {} lease(s) at a time.", - leaseDurationMillis, - epsilonMillis, - renewerIntervalMillis, - takerIntervalMillis, - maxLeasesForWorker, - maxLeasesToStealAtOneTime); - } - - private class TakerRunnable implements Runnable { - - @Override - public void run() { - try { - runTaker(); - } catch (LeasingException e) { - log.error("LeasingException encountered in lease taking thread", e); - } catch (Throwable t) { - log.error("Throwable encountered in lease taking thread", t); - } - } - - } - - private class RenewerRunnable implements Runnable { - - @Override - public void run() { - try { - runRenewer(); - } catch (LeasingException e) { - log.error("LeasingException encountered in lease renewing thread", e); - } catch (Throwable t) { - log.error("Throwable encountered in lease renewing thread", t); - } - } - - } + void initialize() throws ProvisionedThroughputException, DependencyException, IllegalStateException; /** * Start background LeaseHolder and LeaseTaker threads. @@ -188,24 +41,7 @@ public class LeaseCoordinator { * @throws InvalidStateException If the lease table doesn't exist * @throws DependencyException If we encountered exception taking to DynamoDB */ - public void start() throws DependencyException, InvalidStateException, ProvisionedThroughputException { - leaseRenewer.initialize(); - - // 2 because we know we'll have at most 2 concurrent tasks at a time. - leaseCoordinatorThreadPool = Executors.newScheduledThreadPool(2, LEASE_COORDINATOR_THREAD_FACTORY); - - // Taker runs with fixed DELAY because we want it to run slower in the event of performance degredation. - takerFuture = leaseCoordinatorThreadPool.scheduleWithFixedDelay(new TakerRunnable(), - 0L, - takerIntervalMillis, - TimeUnit.MILLISECONDS); - // Renewer runs at fixed INTERVAL because we want it to run at the same rate in the event of degredation. - leaseCoordinatorThreadPool.scheduleAtFixedRate(new RenewerRunnable(), - 0L, - renewerIntervalMillis, - TimeUnit.MILLISECONDS); - running = true; - } + void start() throws DependencyException, InvalidStateException, ProvisionedThroughputException; /** * Runs a single iteration of the lease taker - used by integration tests. @@ -213,28 +49,7 @@ public class LeaseCoordinator { * @throws InvalidStateException * @throws DependencyException */ - protected void runTaker() throws DependencyException, InvalidStateException { - IMetricsScope scope = MetricsHelper.startScope(metricsFactory, "TakeLeases"); - long startTime = System.currentTimeMillis(); - boolean success = false; - - try { - Map takenLeases = leaseTaker.takeLeases(); - - // Only add taken leases to renewer if coordinator is still running. - synchronized (shutdownLock) { - if (running) { - leaseRenewer.addLeasesToRenew(takenLeases.values()); - } - } - - success = true; - } finally { - scope.addDimension(WORKER_IDENTIFIER_METRIC, getWorkerIdentifier()); - MetricsHelper.addSuccessAndLatency(startTime, success, MetricsLevel.SUMMARY); - MetricsHelper.endScope(); - } - } + void runLeaseTaker() throws DependencyException, InvalidStateException; /** * Runs a single iteration of the lease renewer - used by integration tests. @@ -242,108 +57,40 @@ public class LeaseCoordinator { * @throws InvalidStateException * @throws DependencyException */ - protected void runRenewer() throws DependencyException, InvalidStateException { - IMetricsScope scope = MetricsHelper.startScope(metricsFactory, "RenewAllLeases"); - long startTime = System.currentTimeMillis(); - boolean success = false; + void runLeaseRenewer() throws DependencyException, InvalidStateException; - try { - leaseRenewer.renewLeases(); - success = true; - } finally { - scope.addDimension(WORKER_IDENTIFIER_METRIC, getWorkerIdentifier()); - MetricsHelper.addSuccessAndLatency(startTime, success, MetricsLevel.SUMMARY); - MetricsHelper.endScope(); - } - } + /** + * @return true if this LeaseCoordinator is running + */ + boolean isRunning(); + + /** + * @return workerIdentifier + */ + String workerIdentifier(); + + /** + * @return {@link LeaseRefresher} + */ + LeaseRefresher leaseRefresher(); /** * @return currently held leases */ - public Collection getAssignments() { - return leaseRenewer.getCurrentlyHeldLeases().values(); - } + Collection getAssignments(); /** * @param leaseKey lease key to fetch currently held lease for * * @return deep copy of currently held Lease for given key, or null if we don't hold the lease for that key */ - public T getCurrentlyHeldLease(String leaseKey) { - return leaseRenewer.getCurrentlyHeldLease(leaseKey); - } - - /** - * @return workerIdentifier - */ - public String getWorkerIdentifier() { - return leaseTaker.getWorkerIdentifier(); - } - - /** - * Stops background threads and waits for {@link #STOP_WAIT_TIME_MILLIS} for all background tasks to complete. - * If tasks are not completed after this time, method will shutdown thread pool forcefully and return. - */ - public void stop() { - if (leaseCoordinatorThreadPool != null) { - leaseCoordinatorThreadPool.shutdown(); - try { - if (leaseCoordinatorThreadPool.awaitTermination(STOP_WAIT_TIME_MILLIS, TimeUnit.MILLISECONDS)) { - log.info("Worker {} has successfully stopped lease-tracking threads", - leaseTaker.getWorkerIdentifier()); - } else { - leaseCoordinatorThreadPool.shutdownNow(); - log.info("Worker {} stopped lease-tracking threads {} ms after stop", - leaseTaker.getWorkerIdentifier(), - STOP_WAIT_TIME_MILLIS); - } - } catch (InterruptedException e) { - log.debug("Encountered InterruptedException when awaiting threadpool termination"); - } - } else { - log.debug("Threadpool was null, no need to shutdown/terminate threadpool."); - } - - leaseRenewalThreadpool.shutdownNow(); - synchronized (shutdownLock) { - leaseRenewer.clearCurrentlyHeldLeases(); - running = false; - } - } - - /** - * Requests the cancellation of the lease taker. - */ - public void stopLeaseTaker() { - takerFuture.cancel(false); - - } - - /** - * Requests that renewals for the given lease are stopped. - * - * @param lease the lease to stop renewing. - */ - public void dropLease(T lease) { - synchronized (shutdownLock) { - if (lease != null) { - leaseRenewer.dropLease(lease); - } - } - } - - /** - * @return true if this LeaseCoordinator is running - */ - public boolean isRunning() { - return running; - } + Lease getCurrentlyHeldLease(String leaseKey); /** * Updates application-specific lease values in DynamoDB. * * @param lease lease object containing updated values - * @param concurrencyToken obtained by calling Lease.getConcurrencyToken for a currently held lease + * @param concurrencyToken obtained by calling Lease.concurrencyToken for a currently held lease * * @return true if update succeeded, false otherwise * @@ -351,20 +98,43 @@ public class LeaseCoordinator { * @throws ProvisionedThroughputException if DynamoDB update fails due to lack of capacity * @throws DependencyException if DynamoDB update fails in an unexpected way */ - public boolean updateLease(T lease, UUID concurrencyToken) - throws DependencyException, InvalidStateException, ProvisionedThroughputException { - return leaseRenewer.updateLease(lease, concurrencyToken); - } + boolean updateLease(Lease lease, UUID concurrencyToken) + throws DependencyException, InvalidStateException, ProvisionedThroughputException; /** - * Returns executor service that should be used for lease renewal. - * @param maximumPoolSize Maximum allowed thread pool size - * @return Executor service that should be used for lease renewal. + * Requests the cancellation of the lease taker. */ - private static ExecutorService getLeaseRenewalExecutorService(int maximumPoolSize) { - int coreLeaseCount = Math.max(maximumPoolSize / 4, 2); + void stopLeaseTaker(); - return new ThreadPoolExecutor(coreLeaseCount, maximumPoolSize, 60, TimeUnit.SECONDS, - new LinkedTransferQueue(), LEASE_RENEWAL_THREAD_FACTORY); - } + /** + * Requests that renewals for the given lease are stopped. + * + * @param lease the lease to stop renewing. + */ + void dropLease(Lease lease); + + /** + * Stops background threads and waits for specific amount of time for all background tasks to complete. + * If tasks are not completed after this time, method will shutdown thread pool forcefully and return. + */ + void stop(); + + /** + * @return Current shard/lease assignments + */ + List getCurrentAssignments(); + + /** + * @param writeCapacity The DynamoDB table used for tracking leases will be provisioned with the specified initial + * write capacity + * @return LeaseCoordinator + */ + DynamoDBLeaseCoordinator initialLeaseTableWriteCapacity(long writeCapacity); + + /** + * @param readCapacity The DynamoDB table used for tracking leases will be provisioned with the specified initial + * read capacity + * @return LeaseCoordinator + */ + DynamoDBLeaseCoordinator initialLeaseTableReadCapacity(long readCapacity); } diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseManagementConfig.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseManagementConfig.java index f41c2beb..8a9af9d7 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseManagementConfig.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseManagementConfig.java @@ -30,6 +30,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import lombok.Data; import lombok.NonNull; import lombok.experimental.Accessors; +import software.amazon.kinesis.leases.dynamodb.DynamoDBLeaseManagementFactory; import software.amazon.kinesis.metrics.IMetricsFactory; import software.amazon.kinesis.metrics.NullMetricsFactory; diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseManagementFactory.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseManagementFactory.java index f17b0065..e35d3a69 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseManagementFactory.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseManagementFactory.java @@ -15,6 +15,8 @@ package software.amazon.kinesis.leases; +import software.amazon.kinesis.leases.dynamodb.DynamoDBLeaseRefresher; + /** * */ @@ -23,9 +25,7 @@ public interface LeaseManagementFactory { ShardSyncTaskManager createShardSyncTaskManager(); - DynamoDBLeaseManager createLeaseManager(); + DynamoDBLeaseRefresher createLeaseRefresher(); - KinesisClientLibLeaseCoordinator createKinesisClientLibLeaseCoordinator(); - - LeaseManagerProxy createLeaseManagerProxy(); + ShardDetector createShardDetector(); } diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseManager.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseRefresher.java similarity index 80% rename from amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseManager.java rename to amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseRefresher.java index 37415108..f12c5afb 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseManager.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseRefresher.java @@ -19,14 +19,12 @@ import java.util.List; import software.amazon.kinesis.leases.exceptions.DependencyException; import software.amazon.kinesis.leases.exceptions.InvalidStateException; import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException; -import software.amazon.kinesis.leases.Lease; +import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; /** * Supports basic CRUD operations for Leases. - * - * @param Lease subclass, possibly Lease itself. */ -public interface LeaseManager { +public interface LeaseRefresher { /** * Creates the table that will store leases. Succeeds if table already exists. @@ -40,7 +38,7 @@ public interface LeaseManager { * restrictions. * @throws DependencyException if DynamoDB createTable fails in an unexpected way */ - public boolean createLeaseTableIfNotExists(Long readCapacity, Long writeCapacity) + boolean createLeaseTableIfNotExists(Long readCapacity, Long writeCapacity) throws ProvisionedThroughputException, DependencyException; /** @@ -48,7 +46,7 @@ public interface LeaseManager { * * @throws DependencyException if DynamoDB describeTable fails in an unexpected way */ - public boolean leaseTableExists() throws DependencyException; + boolean leaseTableExists() throws DependencyException; /** * Blocks until the lease table exists by polling leaseTableExists. @@ -60,7 +58,7 @@ public interface LeaseManager { * * @throws DependencyException if DynamoDB describeTable fails in an unexpected way */ - public boolean waitUntilLeaseTableExists(long secondsBetweenPolls, long timeoutSeconds) throws DependencyException; + boolean waitUntilLeaseTableExists(long secondsBetweenPolls, long timeoutSeconds) throws DependencyException; /** * List all objects in table synchronously. @@ -71,7 +69,7 @@ public interface LeaseManager { * * @return list of leases */ - public List listLeases() throws DependencyException, InvalidStateException, ProvisionedThroughputException; + List listLeases() throws DependencyException, InvalidStateException, ProvisionedThroughputException; /** * Create a new lease. Conditional on a lease not already existing with this shardId. @@ -84,7 +82,7 @@ public interface LeaseManager { * @throws InvalidStateException if lease table does not exist * @throws ProvisionedThroughputException if DynamoDB put fails due to lack of capacity */ - public boolean createLeaseIfNotExists(T lease) + boolean createLeaseIfNotExists(Lease lease) throws DependencyException, InvalidStateException, ProvisionedThroughputException; /** @@ -96,7 +94,7 @@ public interface LeaseManager { * * @return lease for the specified shardId, or null if one doesn't exist */ - public T getLease(String shardId) throws DependencyException, InvalidStateException, ProvisionedThroughputException; + Lease getLease(String shardId) throws DependencyException, InvalidStateException, ProvisionedThroughputException; /** * Renew a lease by incrementing the lease counter. Conditional on the leaseCounter in DynamoDB matching the leaseCounter @@ -110,7 +108,7 @@ public interface LeaseManager { * @throws ProvisionedThroughputException if DynamoDB update fails due to lack of capacity * @throws DependencyException if DynamoDB update fails in an unexpected way */ - public boolean renewLease(T lease) + boolean renewLease(Lease lease) throws DependencyException, InvalidStateException, ProvisionedThroughputException; /** @@ -127,7 +125,7 @@ public interface LeaseManager { * @throws ProvisionedThroughputException if DynamoDB update fails due to lack of capacity * @throws DependencyException if DynamoDB update fails in an unexpected way */ - public boolean takeLease(T lease, String owner) + boolean takeLease(Lease lease, String owner) throws DependencyException, InvalidStateException, ProvisionedThroughputException; /** @@ -142,7 +140,7 @@ public interface LeaseManager { * @throws ProvisionedThroughputException if DynamoDB update fails due to lack of capacity * @throws DependencyException if DynamoDB update fails in an unexpected way */ - public boolean evictLease(T lease) + boolean evictLease(Lease lease) throws DependencyException, InvalidStateException, ProvisionedThroughputException; /** @@ -154,7 +152,7 @@ public interface LeaseManager { * @throws ProvisionedThroughputException if DynamoDB delete fails due to lack of capacity * @throws DependencyException if DynamoDB delete fails in an unexpected way */ - public void deleteLease(T lease) throws DependencyException, InvalidStateException, ProvisionedThroughputException; + void deleteLease(Lease lease) throws DependencyException, InvalidStateException, ProvisionedThroughputException; /** * Delete all leases from DynamoDB. Useful for tools/utils and testing. @@ -163,7 +161,7 @@ public interface LeaseManager { * @throws ProvisionedThroughputException if DynamoDB scan or delete fail due to lack of capacity * @throws DependencyException if DynamoDB scan or delete fail in an unexpected way */ - public void deleteAll() throws DependencyException, InvalidStateException, ProvisionedThroughputException; + void deleteAll() throws DependencyException, InvalidStateException, ProvisionedThroughputException; /** * Update application-specific fields of the given lease in DynamoDB. Does not update fields managed by the leasing @@ -177,7 +175,7 @@ public interface LeaseManager { * @throws ProvisionedThroughputException if DynamoDB update fails due to lack of capacity * @throws DependencyException if DynamoDB update fails in an unexpected way */ - public boolean updateLease(T lease) + boolean updateLease(Lease lease) throws DependencyException, InvalidStateException, ProvisionedThroughputException; /** @@ -189,6 +187,20 @@ public interface LeaseManager { * @throws InvalidStateException if lease table does not exist * @throws ProvisionedThroughputException if DynamoDB scan fails due to lack of capacity */ - public boolean isLeaseTableEmpty() throws DependencyException, InvalidStateException, ProvisionedThroughputException; + boolean isLeaseTableEmpty() throws DependencyException, InvalidStateException, ProvisionedThroughputException; + + /** + * Gets the current checkpoint of the shard. This is useful in the resharding use case + * where we will wait for the parent shard to complete before starting on the records from a child shard. + * + * @param shardId Checkpoint of this shard will be returned + * @return Checkpoint of this shard, or null if the shard record doesn't exist. + * + * @throws ProvisionedThroughputException if DynamoDB update fails due to lack of capacity + * @throws InvalidStateException if lease table does not exist + * @throws DependencyException if DynamoDB update fails in an unexpected way + */ + ExtendedSequenceNumber getCheckpoint(String shardId) + throws ProvisionedThroughputException, InvalidStateException, DependencyException; } diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseRenewer.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseRenewer.java index aaf5b6d0..e1b0f272 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseRenewer.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseRenewer.java @@ -21,22 +21,21 @@ import java.util.UUID; import software.amazon.kinesis.leases.exceptions.DependencyException; import software.amazon.kinesis.leases.exceptions.InvalidStateException; import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException; -import software.amazon.kinesis.leases.Lease; /** * ILeaseRenewer objects are used by LeaseCoordinator to renew leases held by the LeaseCoordinator. Each * LeaseCoordinator instance corresponds to one worker, and uses exactly one ILeaseRenewer to manage lease renewal for * that worker. */ -public interface LeaseRenewer { +public interface LeaseRenewer { /** - * Bootstrap initial set of leases from the LeaseManager (e.g. upon process restart, pick up leases we own) + * Bootstrap initial set of leases from the {@link LeaseRefresher} (e.g. upon process restart, pick up leases we own) * @throws DependencyException on unexpected DynamoDB failures * @throws InvalidStateException if lease table doesn't exist * @throws ProvisionedThroughputException if DynamoDB reads fail due to insufficient capacity */ - public void initialize() throws DependencyException, InvalidStateException, ProvisionedThroughputException; + void initialize() throws DependencyException, InvalidStateException, ProvisionedThroughputException; /** * Attempt to renew all currently held leases. @@ -44,21 +43,21 @@ public interface LeaseRenewer { * @throws DependencyException on unexpected DynamoDB failures * @throws InvalidStateException if lease table does not exist */ - public void renewLeases() throws DependencyException, InvalidStateException; + void renewLeases() throws DependencyException, InvalidStateException; /** * @return currently held leases. Key is shardId, value is corresponding Lease object. A lease is currently held if * we successfully renewed it on the last run of renewLeases(). Lease objects returned are deep copies - * their lease counters will not tick. */ - public Map getCurrentlyHeldLeases(); + Map getCurrentlyHeldLeases(); /** * @param leaseKey key of the lease to retrieve * * @return a deep copy of a currently held lease, or null if we don't hold the lease */ - public T getCurrentlyHeldLease(String leaseKey); + Lease getCurrentlyHeldLease(String leaseKey); /** * Adds leases to this LeaseRenewer's set of currently held leases. Leases must have lastRenewalNanos set to the @@ -66,19 +65,19 @@ public interface LeaseRenewer { * * @param newLeases new leases. */ - public void addLeasesToRenew(Collection newLeases); + void addLeasesToRenew(Collection newLeases); /** * Clears this LeaseRenewer's set of currently held leases. */ - public void clearCurrentlyHeldLeases(); + void clearCurrentlyHeldLeases(); /** * Stops the lease renewer from continunig to maintain the given lease. * * @param lease the lease to drop. */ - void dropLease(T lease); + void dropLease(Lease lease); /** * Update application-specific fields in a currently held lease. Cannot be used to update internal fields such as @@ -86,7 +85,7 @@ public interface LeaseRenewer { * the concurrency token on the internal authoritative copy of the lease (ie, if we lost and re-acquired the lease). * * @param lease lease object containing updated data - * @param concurrencyToken obtained by calling Lease.getConcurrencyToken for a currently held lease + * @param concurrencyToken obtained by calling Lease.concurrencyToken for a currently held lease * * @return true if update succeeds, false otherwise * @@ -94,7 +93,7 @@ public interface LeaseRenewer { * @throws ProvisionedThroughputException if DynamoDB update fails due to lack of capacity * @throws DependencyException if DynamoDB update fails in an unexpected way */ - boolean updateLease(T lease, UUID concurrencyToken) + boolean updateLease(Lease lease, UUID concurrencyToken) throws DependencyException, InvalidStateException, ProvisionedThroughputException; } diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseSerializer.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseSerializer.java index 15e75952..f3f9b5c8 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseSerializer.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseSerializer.java @@ -26,10 +26,8 @@ import software.amazon.kinesis.leases.Lease; /** * Utility class that manages the mapping of Lease objects/operations to records in DynamoDB. - * - * @param Lease subclass, possibly Lease itself */ -public interface LeaseSerializer { +public interface LeaseSerializer { /** * Construct a DynamoDB record out of a Lease object @@ -37,7 +35,7 @@ public interface LeaseSerializer { * @param lease lease object to serialize * @return an attribute value map representing the lease object */ - public Map toDynamoRecord(T lease); + Map toDynamoRecord(Lease lease); /** * Construct a Lease object out of a DynamoDB record. @@ -45,72 +43,72 @@ public interface LeaseSerializer { * @param dynamoRecord attribute value map from DynamoDB * @return a deserialized lease object representing the attribute value map */ - public T fromDynamoRecord(Map dynamoRecord); + Lease fromDynamoRecord(Map dynamoRecord); /** * @param lease * @return the attribute value map representing a Lease's hash key given a Lease object. */ - public Map getDynamoHashKey(T lease); + Map getDynamoHashKey(Lease lease); /** - * Special getDynamoHashKey implementation used by ILeaseManager.getLease(). + * Special getDynamoHashKey implementation used by {@link LeaseRefresher#getLease(String)}. * * @param leaseKey * @return the attribute value map representing a Lease's hash key given a string. */ - public Map getDynamoHashKey(String leaseKey); + Map getDynamoHashKey(String leaseKey); /** * @param lease * @return the attribute value map asserting that a lease counter is what we expect. */ - public Map getDynamoLeaseCounterExpectation(T lease); + Map getDynamoLeaseCounterExpectation(Lease lease); /** * @param lease * @return the attribute value map asserting that the lease owner is what we expect. */ - public Map getDynamoLeaseOwnerExpectation(T lease); + Map getDynamoLeaseOwnerExpectation(Lease lease); /** * @return the attribute value map asserting that a lease does not exist. */ - public Map getDynamoNonexistantExpectation(); + Map getDynamoNonexistantExpectation(); /** * @param lease * @return the attribute value map that increments a lease counter */ - public Map getDynamoLeaseCounterUpdate(T lease); + Map getDynamoLeaseCounterUpdate(Lease lease); /** * @param lease * @param newOwner * @return the attribute value map that takes a lease for a new owner */ - public Map getDynamoTakeLeaseUpdate(T lease, String newOwner); + Map getDynamoTakeLeaseUpdate(Lease lease, String newOwner); /** * @param lease * @return the attribute value map that voids a lease */ - public Map getDynamoEvictLeaseUpdate(T lease); + Map getDynamoEvictLeaseUpdate(Lease lease); /** * @param lease * @return the attribute value map that updates application-specific data for a lease and increments the lease * counter */ - public Map getDynamoUpdateLeaseUpdate(T lease); + Map getDynamoUpdateLeaseUpdate(Lease lease); /** * @return the key schema for creating a DynamoDB table to store leases */ - public Collection getKeySchema(); + Collection getKeySchema(); /** * @return attribute definitions for creating a DynamoDB table to store leases */ - public Collection getAttributeDefinitions(); + Collection getAttributeDefinitions(); } diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseTaker.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseTaker.java index e54533b3..9d00ff17 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseTaker.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseTaker.java @@ -18,13 +18,12 @@ import java.util.Map; import software.amazon.kinesis.leases.exceptions.DependencyException; import software.amazon.kinesis.leases.exceptions.InvalidStateException; -import software.amazon.kinesis.leases.Lease; /** * ILeaseTaker is used by LeaseCoordinator to take new leases, or leases that other workers fail to renew. Each * LeaseCoordinator instance corresponds to one worker and uses exactly one ILeaseTaker to take leases for that worker. */ -public interface LeaseTaker { +public interface LeaseTaker { /** * Compute the set of leases available to be taken and attempt to take them. Lease taking rules are: @@ -39,11 +38,11 @@ public interface LeaseTaker { * @throws DependencyException on unexpected DynamoDB failures * @throws InvalidStateException if lease table does not exist */ - public abstract Map takeLeases() throws DependencyException, InvalidStateException; + Map takeLeases() throws DependencyException, InvalidStateException; /** * @return workerIdentifier for this LeaseTaker */ - public abstract String getWorkerIdentifier(); + String getWorkerIdentifier(); } diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseManagerProxy.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ShardDetector.java similarity index 95% rename from amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseManagerProxy.java rename to amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ShardDetector.java index de8200b8..6f7f5a00 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseManagerProxy.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ShardDetector.java @@ -22,7 +22,7 @@ import java.util.List; /** * */ -public interface LeaseManagerProxy { +public interface ShardDetector { List listShards(); } diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ShardSyncTask.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ShardSyncTask.java index 7ed43518..19f92a83 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ShardSyncTask.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ShardSyncTask.java @@ -34,9 +34,9 @@ import software.amazon.kinesis.lifecycle.TaskType; @Slf4j public class ShardSyncTask implements ITask { @NonNull - private final LeaseManagerProxy leaseManagerProxy; + private final ShardDetector shardDetector; @NonNull - private final LeaseManager leaseManager; + private final LeaseRefresher leaseRefresher; @NonNull private final InitialPositionInStreamExtended initialPosition; private final boolean cleanupLeasesUponShardCompletion; @@ -56,7 +56,7 @@ public class ShardSyncTask implements ITask { Exception exception = null; try { - ShardSyncer.checkAndCreateLeasesForNewShards(leaseManagerProxy, leaseManager, initialPosition, + ShardSyncer.checkAndCreateLeasesForNewShards(shardDetector, leaseRefresher, initialPosition, cleanupLeasesUponShardCompletion, ignoreUnexpectedChildShards); if (shardSyncTaskIdleTimeMillis > 0) { Thread.sleep(shardSyncTaskIdleTimeMillis); diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ShardSyncTaskManager.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ShardSyncTaskManager.java index 591dda20..66ba14bb 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ShardSyncTaskManager.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ShardSyncTaskManager.java @@ -39,9 +39,9 @@ import software.amazon.kinesis.metrics.MetricsCollectingTaskDecorator; @Slf4j public class ShardSyncTaskManager { @NonNull - private final LeaseManagerProxy leaseManagerProxy; + private final ShardDetector shardDetector; @NonNull - private final LeaseManager leaseManager; + private final LeaseRefresher leaseRefresher; @NonNull private final InitialPositionInStreamExtended initialPositionInStream; private final boolean cleanupLeasesUponShardCompletion; @@ -76,8 +76,8 @@ public class ShardSyncTaskManager { currentTask = new MetricsCollectingTaskDecorator( - new ShardSyncTask(leaseManagerProxy, - leaseManager, + new ShardSyncTask(shardDetector, + leaseRefresher, initialPositionInStream, cleanupLeasesUponShardCompletion, ignoreUnexpectedChildShards, diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ShardSyncer.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ShardSyncer.java index 233f7d34..a44bbf3d 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ShardSyncer.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/ShardSyncer.java @@ -57,20 +57,20 @@ public class ShardSyncer { private ShardSyncer() { } - static synchronized void bootstrapShardLeases(@NonNull final LeaseManagerProxy leaseManagerProxy, - @NonNull final LeaseManager leaseManager, + static synchronized void bootstrapShardLeases(@NonNull final ShardDetector shardDetector, + @NonNull final LeaseRefresher leaseRefresher, @NonNull final InitialPositionInStreamExtended initialPositionInStream, final boolean cleanupLeasesOfCompletedShards, final boolean ignoreUnexpectedChildShards) throws DependencyException, InvalidStateException, ProvisionedThroughputException, KinesisClientLibIOException { - syncShardLeases(leaseManagerProxy, leaseManager, initialPositionInStream, cleanupLeasesOfCompletedShards, + syncShardLeases(shardDetector, leaseRefresher, initialPositionInStream, cleanupLeasesOfCompletedShards, ignoreUnexpectedChildShards); } /** * Check and create leases for any new shards (e.g. following a reshard operation). * - * @param leaseManager + * @param leaseRefresher * @param initialPositionInStream * @param cleanupLeasesOfCompletedShards * @param ignoreUnexpectedChildShards @@ -79,29 +79,29 @@ public class ShardSyncer { * @throws ProvisionedThroughputException * @throws KinesisClientLibIOException */ - public static synchronized void checkAndCreateLeasesForNewShards(@NonNull final LeaseManagerProxy leaseManagerProxy, - @NonNull final LeaseManager leaseManager, + public static synchronized void checkAndCreateLeasesForNewShards(@NonNull final ShardDetector shardDetector, + @NonNull final LeaseRefresher leaseRefresher, @NonNull final InitialPositionInStreamExtended initialPositionInStream, final boolean cleanupLeasesOfCompletedShards, final boolean ignoreUnexpectedChildShards) throws DependencyException, InvalidStateException, ProvisionedThroughputException, KinesisClientLibIOException { - syncShardLeases(leaseManagerProxy, leaseManager, initialPositionInStream, + syncShardLeases(shardDetector, leaseRefresher, initialPositionInStream, cleanupLeasesOfCompletedShards, ignoreUnexpectedChildShards); } - static synchronized void checkAndCreateLeasesForNewShards(@NonNull final LeaseManagerProxy leaseManagerProxy, - @NonNull final LeaseManager leaseManager, + static synchronized void checkAndCreateLeasesForNewShards(@NonNull final ShardDetector shardDetector, + @NonNull final LeaseRefresher leaseRefresher, @NonNull final InitialPositionInStreamExtended initialPositionInStream, final boolean cleanupLeasesOfCompletedShards) throws DependencyException, InvalidStateException, ProvisionedThroughputException, KinesisClientLibIOException { - checkAndCreateLeasesForNewShards(leaseManagerProxy, leaseManager, initialPositionInStream, + checkAndCreateLeasesForNewShards(shardDetector, leaseRefresher, initialPositionInStream, cleanupLeasesOfCompletedShards, false); } /** * Sync leases with Kinesis shards (e.g. at startup, or when we reach end of a shard). * - * @param leaseManager + * @param leaseRefresher * @param initialPosition * @param cleanupLeasesOfCompletedShards * @param ignoreUnexpectedChildShards @@ -111,13 +111,13 @@ public class ShardSyncer { * @throws KinesisClientLibIOException */ // CHECKSTYLE:OFF CyclomaticComplexity - private static synchronized void syncShardLeases(@NonNull final LeaseManagerProxy leaseManagerProxy, - final LeaseManager leaseManager, + private static synchronized void syncShardLeases(@NonNull final ShardDetector shardDetector, + final LeaseRefresher leaseRefresher, final InitialPositionInStreamExtended initialPosition, final boolean cleanupLeasesOfCompletedShards, final boolean ignoreUnexpectedChildShards) throws DependencyException, InvalidStateException, ProvisionedThroughputException, KinesisClientLibIOException { - List shards = getShardList(leaseManagerProxy); + List shards = getShardList(shardDetector); log.debug("Num shards: {}", shards.size()); Map shardIdToShardMap = constructShardIdToShardMap(shards); @@ -127,34 +127,34 @@ public class ShardSyncer { assertAllParentShardsAreClosed(inconsistentShardIds); } - List currentLeases = leaseManager.listLeases(); + List currentLeases = leaseRefresher.listLeases(); - List newLeasesToCreate = determineNewLeasesToCreate(shards, currentLeases, initialPosition, - inconsistentShardIds); + List newLeasesToCreate = determineNewLeasesToCreate(shards, currentLeases, initialPosition, + inconsistentShardIds); log.debug("Num new leases to create: {}", newLeasesToCreate.size()); - for (KinesisClientLease lease : newLeasesToCreate) { + for (Lease lease : newLeasesToCreate) { long startTimeMillis = System.currentTimeMillis(); boolean success = false; try { - leaseManager.createLeaseIfNotExists(lease); + leaseRefresher.createLeaseIfNotExists(lease); success = true; } finally { MetricsHelper.addSuccessAndLatency("CreateLease", startTimeMillis, success, MetricsLevel.DETAILED); } } - List trackedLeases = new ArrayList<>(); + List trackedLeases = new ArrayList<>(); if (currentLeases != null) { trackedLeases.addAll(currentLeases); } trackedLeases.addAll(newLeasesToCreate); - cleanupGarbageLeases(leaseManagerProxy, shards, trackedLeases, leaseManager); + cleanupGarbageLeases(shardDetector, shards, trackedLeases, leaseRefresher); if (cleanupLeasesOfCompletedShards) { cleanupLeasesOfFinishedShards(currentLeases, shardIdToShardMap, shardIdToChildShardIdsMap, trackedLeases, - leaseManager); + leaseRefresher); } } // CHECKSTYLE:ON CyclomaticComplexity @@ -200,10 +200,10 @@ public class ShardSyncer { * @param trackedLeaseList * @return */ - static Map constructShardIdToKCLLeaseMap(List trackedLeaseList) { - Map trackedLeasesMap = new HashMap<>(); - for (KinesisClientLease lease : trackedLeaseList) { - trackedLeasesMap.put(lease.getLeaseKey(), lease); + static Map constructShardIdToKCLLeaseMap(List trackedLeaseList) { + Map trackedLeasesMap = new HashMap<>(); + for (Lease lease : trackedLeaseList) { + trackedLeasesMap.put(lease.leaseKey(), lease); } return trackedLeasesMap; } @@ -313,8 +313,8 @@ public class ShardSyncer { return shardIdToChildShardIdsMap; } - private static List getShardList(@NonNull final LeaseManagerProxy leaseManagerProxy) throws KinesisClientLibIOException { - List shards = leaseManagerProxy.listShards(); + private static List getShardList(@NonNull final ShardDetector shardDetector) throws KinesisClientLibIOException { + List shards = shardDetector.listShards(); if (shards == null) { throw new KinesisClientLibIOException( "Stream is not in ACTIVE OR UPDATING state - will retry getting the shard list."); @@ -369,16 +369,16 @@ public class ShardSyncer { * @param inconsistentShardIds Set of child shard ids having open parents. * @return List of new leases to create sorted by starting sequenceNumber of the corresponding shard */ - static List determineNewLeasesToCreate(List shards, - List currentLeases, + static List determineNewLeasesToCreate(List shards, + List currentLeases, InitialPositionInStreamExtended initialPosition, Set inconsistentShardIds) { - Map shardIdToNewLeaseMap = new HashMap(); + Map shardIdToNewLeaseMap = new HashMap<>(); Map shardIdToShardMapOfAllKinesisShards = constructShardIdToShardMap(shards); - Set shardIdsOfCurrentLeases = new HashSet(); - for (KinesisClientLease lease : currentLeases) { - shardIdsOfCurrentLeases.add(lease.getLeaseKey()); + Set shardIdsOfCurrentLeases = new HashSet<>(); + for (Lease lease : currentLeases) { + shardIdsOfCurrentLeases.add(lease.leaseKey()); log.debug("Existing lease: {}", lease); } @@ -395,7 +395,7 @@ public class ShardSyncer { log.info("shardId {} is an inconsistent child. Not creating a lease", shardId); } else { log.debug("Need to create a lease for shardId {}", shardId); - KinesisClientLease newLease = newKCLLease(shard); + Lease newLease = newKCLLease(shard); boolean isDescendant = checkIfDescendantAndAddNewLeasesForAncestors(shardId, initialPosition, @@ -429,18 +429,18 @@ public class ShardSyncer { */ if (isDescendant && !initialPosition.getInitialPositionInStream() .equals(InitialPositionInStream.AT_TIMESTAMP)) { - newLease.setCheckpoint(ExtendedSequenceNumber.TRIM_HORIZON); + newLease.checkpoint(ExtendedSequenceNumber.TRIM_HORIZON); } else { - newLease.setCheckpoint(convertToCheckpoint(initialPosition)); + newLease.checkpoint(convertToCheckpoint(initialPosition)); } - log.debug("Set checkpoint of {} to {}", newLease.getLeaseKey(), newLease.getCheckpoint()); + log.debug("Set checkpoint of {} to {}", newLease.leaseKey(), newLease.checkpoint()); shardIdToNewLeaseMap.put(shardId, newLease); } } - List newLeasesToCreate = new ArrayList(); + List newLeasesToCreate = new ArrayList<>(); newLeasesToCreate.addAll(shardIdToNewLeaseMap.values()); - Comparator startingSequenceNumberComparator = + Comparator startingSequenceNumberComparator = new StartingSequenceNumberAndShardIdBasedComparator(shardIdToShardMapOfAllKinesisShards); Collections.sort(newLeasesToCreate, startingSequenceNumberComparator); return newLeasesToCreate; @@ -450,10 +450,10 @@ public class ShardSyncer { * Determine new leases to create and their initial checkpoint. * Note: Package level access only for testing purposes. */ - static List determineNewLeasesToCreate(List shards, - List currentLeases, + static List determineNewLeasesToCreate(List shards, + List currentLeases, InitialPositionInStreamExtended initialPosition) { - Set inconsistentShardIds = new HashSet(); + Set inconsistentShardIds = new HashSet<>(); return determineNewLeasesToCreate(shards, currentLeases, initialPosition, inconsistentShardIds); } @@ -477,7 +477,7 @@ public class ShardSyncer { InitialPositionInStreamExtended initialPosition, Set shardIdsOfCurrentLeases, Map shardIdToShardMapOfAllKinesisShards, - Map shardIdToLeaseMapOfNewShards, + Map shardIdToLeaseMapOfNewShards, Map memoizationContext) { Boolean previousValue = memoizationContext.get(shardId); @@ -520,7 +520,7 @@ public class ShardSyncer { for (String parentShardId : parentShardIds) { if (!shardIdsOfCurrentLeases.contains(parentShardId)) { log.debug("Need to create a lease for shardId {}", parentShardId); - KinesisClientLease lease = shardIdToLeaseMapOfNewShards.get(parentShardId); + Lease lease = shardIdToLeaseMapOfNewShards.get(parentShardId); if (lease == null) { lease = newKCLLease(shardIdToShardMapOfAllKinesisShards.get(parentShardId)); shardIdToLeaseMapOfNewShards.put(parentShardId, lease); @@ -529,9 +529,9 @@ public class ShardSyncer { if (descendantParentShardIds.contains(parentShardId) && !initialPosition.getInitialPositionInStream() .equals(InitialPositionInStream.AT_TIMESTAMP)) { - lease.setCheckpoint(ExtendedSequenceNumber.TRIM_HORIZON); + lease.checkpoint(ExtendedSequenceNumber.TRIM_HORIZON); } else { - lease.setCheckpoint(convertToCheckpoint(initialPosition)); + lease.checkpoint(convertToCheckpoint(initialPosition)); } } } @@ -584,16 +584,16 @@ public class ShardSyncer { * * the parentShardIds listed in the lease are also not present in the list of Kinesis shards. * @param shards List of all Kinesis shards (assumed to be a consistent snapshot - when stream is in Active state). * @param trackedLeases List of - * @param leaseManager + * @param leaseRefresher * @throws KinesisClientLibIOException Thrown if we couldn't get a fresh shard list from Kinesis. * @throws ProvisionedThroughputException * @throws InvalidStateException * @throws DependencyException */ - private static void cleanupGarbageLeases(@NonNull final LeaseManagerProxy leaseManagerProxy, + private static void cleanupGarbageLeases(@NonNull final ShardDetector shardDetector, final List shards, - final List trackedLeases, - final LeaseManager leaseManager) + final List trackedLeases, + final LeaseRefresher leaseRefresher) throws KinesisClientLibIOException, DependencyException, InvalidStateException, ProvisionedThroughputException { Set kinesisShards = new HashSet<>(); for (Shard shard : shards) { @@ -601,8 +601,8 @@ public class ShardSyncer { } // Check if there are leases for non-existent shards - List garbageLeases = new ArrayList<>(); - for (KinesisClientLease lease : trackedLeases) { + List garbageLeases = new ArrayList<>(); + for (Lease lease : trackedLeases) { if (isCandidateForCleanup(lease, kinesisShards)) { garbageLeases.add(lease); } @@ -611,16 +611,16 @@ public class ShardSyncer { if (!garbageLeases.isEmpty()) { log.info("Found {} candidate leases for cleanup. Refreshing list of" + " Kinesis shards to pick up recent/latest shards", garbageLeases.size()); - List currentShardList = getShardList(leaseManagerProxy); + List currentShardList = getShardList(shardDetector); Set currentKinesisShardIds = new HashSet<>(); for (Shard shard : currentShardList) { currentKinesisShardIds.add(shard.getShardId()); } - for (KinesisClientLease lease : garbageLeases) { + for (Lease lease : garbageLeases) { if (isCandidateForCleanup(lease, currentKinesisShardIds)) { - log.info("Deleting lease for shard {} as it is not present in Kinesis stream.", lease.getLeaseKey()); - leaseManager.deleteLease(lease); + log.info("Deleting lease for shard {} as it is not present in Kinesis stream.", lease.leaseKey()); + leaseRefresher.deleteLease(lease); } } } @@ -637,15 +637,15 @@ public class ShardSyncer { * @throws KinesisClientLibIOException Thrown if currentKinesisShardIds contains a parent shard but not the child * shard (we are evaluating for deletion). */ - static boolean isCandidateForCleanup(KinesisClientLease lease, Set currentKinesisShardIds) + static boolean isCandidateForCleanup(Lease lease, Set currentKinesisShardIds) throws KinesisClientLibIOException { boolean isCandidateForCleanup = true; - if (currentKinesisShardIds.contains(lease.getLeaseKey())) { + if (currentKinesisShardIds.contains(lease.leaseKey())) { isCandidateForCleanup = false; } else { - log.info("Found lease for non-existent shard: {}. Checking its parent shards", lease.getLeaseKey()); - Set parentShardIds = lease.getParentShardIds(); + log.info("Found lease for non-existent shard: {}. Checking its parent shards", lease.leaseKey()); + Set parentShardIds = lease.parentShardIds(); for (String parentShardId : parentShardIds) { // Throw an exception if the parent shard exists (but the child does not). @@ -653,7 +653,7 @@ public class ShardSyncer { if (currentKinesisShardIds.contains(parentShardId)) { String message = "Parent shard " + parentShardId + " exists but not the child shard " - + lease.getLeaseKey(); + + lease.leaseKey(); log.info(message); throw new KinesisClientLibIOException(message); } @@ -674,23 +674,23 @@ public class ShardSyncer { * @param shardIdToShardMap Map of shardId->Shard (assumed to include all Kinesis shards) * @param shardIdToChildShardIdsMap Map of shardId->childShardIds (assumed to include all Kinesis shards) * @param trackedLeases List of all leases we are tracking. - * @param leaseManager Lease manager (will be used to delete leases) + * @param leaseRefresher Lease refresher (will be used to delete leases) * @throws DependencyException * @throws InvalidStateException * @throws ProvisionedThroughputException * @throws KinesisClientLibIOException */ - private static synchronized void cleanupLeasesOfFinishedShards(Collection currentLeases, + private static synchronized void cleanupLeasesOfFinishedShards(Collection currentLeases, Map shardIdToShardMap, Map> shardIdToChildShardIdsMap, - List trackedLeases, - LeaseManager leaseManager) + List trackedLeases, + LeaseRefresher leaseRefresher) throws DependencyException, InvalidStateException, ProvisionedThroughputException, KinesisClientLibIOException { Set shardIdsOfClosedShards = new HashSet<>(); - List leasesOfClosedShards = new ArrayList<>(); - for (KinesisClientLease lease : currentLeases) { - if (lease.getCheckpoint().equals(ExtendedSequenceNumber.SHARD_END)) { - shardIdsOfClosedShards.add(lease.getLeaseKey()); + List leasesOfClosedShards = new ArrayList<>(); + for (Lease lease : currentLeases) { + if (lease.checkpoint().equals(ExtendedSequenceNumber.SHARD_END)) { + shardIdsOfClosedShards.add(lease.leaseKey()); leasesOfClosedShards.add(lease); } } @@ -699,16 +699,16 @@ public class ShardSyncer { assertClosedShardsAreCoveredOrAbsent(shardIdToShardMap, shardIdToChildShardIdsMap, shardIdsOfClosedShards); - Comparator startingSequenceNumberComparator = + Comparator startingSequenceNumberComparator = new StartingSequenceNumberAndShardIdBasedComparator(shardIdToShardMap); Collections.sort(leasesOfClosedShards, startingSequenceNumberComparator); - Map trackedLeaseMap = constructShardIdToKCLLeaseMap(trackedLeases); + Map trackedLeaseMap = constructShardIdToKCLLeaseMap(trackedLeases); - for (KinesisClientLease leaseOfClosedShard : leasesOfClosedShards) { - String closedShardId = leaseOfClosedShard.getLeaseKey(); + for (Lease leaseOfClosedShard : leasesOfClosedShards) { + String closedShardId = leaseOfClosedShard.leaseKey(); Set childShardIds = shardIdToChildShardIdsMap.get(closedShardId); if ((closedShardId != null) && (childShardIds != null) && (!childShardIds.isEmpty())) { - cleanupLeaseForClosedShard(closedShardId, childShardIds, trackedLeaseMap, leaseManager); + cleanupLeaseForClosedShard(closedShardId, childShardIds, trackedLeaseMap, leaseRefresher); } } } @@ -722,33 +722,33 @@ public class ShardSyncer { * * @param closedShardId Identifies the closed shard * @param childShardIds ShardIds of children of the closed shard - * @param trackedLeases shardId->KinesisClientLease map with all leases we are tracking (should not be null) - * @param leaseManager + * @param trackedLeases shardId->Lease map with all leases we are tracking (should not be null) + * @param leaseRefresher * @throws ProvisionedThroughputException * @throws InvalidStateException * @throws DependencyException */ static synchronized void cleanupLeaseForClosedShard(String closedShardId, Set childShardIds, - Map trackedLeases, - LeaseManager leaseManager) + Map trackedLeases, + LeaseRefresher leaseRefresher) throws DependencyException, InvalidStateException, ProvisionedThroughputException { - KinesisClientLease leaseForClosedShard = trackedLeases.get(closedShardId); - List childShardLeases = new ArrayList<>(); + Lease leaseForClosedShard = trackedLeases.get(closedShardId); + List childShardLeases = new ArrayList<>(); for (String childShardId : childShardIds) { - KinesisClientLease childLease = trackedLeases.get(childShardId); + Lease childLease = trackedLeases.get(childShardId); if (childLease != null) { childShardLeases.add(childLease); } } if ((leaseForClosedShard != null) - && (leaseForClosedShard.getCheckpoint().equals(ExtendedSequenceNumber.SHARD_END)) + && (leaseForClosedShard.checkpoint().equals(ExtendedSequenceNumber.SHARD_END)) && (childShardLeases.size() == childShardIds.size())) { boolean okayToDelete = true; - for (KinesisClientLease lease : childShardLeases) { - if (lease.getCheckpoint().equals(ExtendedSequenceNumber.TRIM_HORIZON)) { + for (Lease lease : childShardLeases) { + if (lease.checkpoint().equals(ExtendedSequenceNumber.TRIM_HORIZON)) { okayToDelete = false; break; } @@ -756,22 +756,22 @@ public class ShardSyncer { if (okayToDelete) { log.info("Deleting lease for shard {} as it has been completely processed and processing of child " - + "shards has begun.", leaseForClosedShard.getLeaseKey()); - leaseManager.deleteLease(leaseForClosedShard); + + "shards has begun.", leaseForClosedShard.leaseKey()); + leaseRefresher.deleteLease(leaseForClosedShard); } } } /** - * Helper method to create a new KinesisClientLease POJO for a shard. + * Helper method to create a new Lease POJO for a shard. * Note: Package level access only for testing purposes * * @param shard * @return */ - public static KinesisClientLease newKCLLease(Shard shard) { - KinesisClientLease newLease = new KinesisClientLease(); - newLease.setLeaseKey(shard.getShardId()); + public static Lease newKCLLease(Shard shard) { + Lease newLease = new Lease(); + newLease.leaseKey(shard.getShardId()); List parentShardIds = new ArrayList(2); if (shard.getParentShardId() != null) { parentShardIds.add(shard.getParentShardId()); @@ -779,8 +779,8 @@ public class ShardSyncer { if (shard.getAdjacentParentShardId() != null) { parentShardIds.add(shard.getAdjacentParentShardId()); } - newLease.setParentShardIds(parentShardIds); - newLease.setOwnerSwitchesSinceCheckpoint(0L); + newLease.parentShardIds(parentShardIds); + newLease.ownerSwitchesSinceCheckpoint(0L); return newLease; } @@ -835,8 +835,7 @@ public class ShardSyncer { /** Helper class to compare leases based on starting sequence number of the corresponding shards. * */ - private static class StartingSequenceNumberAndShardIdBasedComparator implements Comparator, - Serializable { + private static class StartingSequenceNumberAndShardIdBasedComparator implements Comparator, Serializable { private static final long serialVersionUID = 1L; @@ -859,10 +858,10 @@ public class ShardSyncer { * {@inheritDoc} */ @Override - public int compare(KinesisClientLease lease1, KinesisClientLease lease2) { + public int compare(Lease lease1, Lease lease2) { int result = 0; - String shardId1 = lease1.getLeaseKey(); - String shardId2 = lease2.getLeaseKey(); + String shardId1 = lease1.leaseKey(); + String shardId2 = lease2.leaseKey(); Shard shard1 = shardIdToShardMap.get(shardId1); Shard shard2 = shardIdToShardMap.get(shardId2); diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseCoordinator.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseCoordinator.java new file mode 100644 index 00000000..109c16f6 --- /dev/null +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseCoordinator.java @@ -0,0 +1,357 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package software.amazon.kinesis.leases.dynamodb; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedTransferQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import com.amazonaws.util.CollectionUtils; +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +import lombok.extern.slf4j.Slf4j; +import software.amazon.kinesis.leases.Lease; +import software.amazon.kinesis.leases.LeaseCoordinator; +import software.amazon.kinesis.leases.LeaseRefresher; +import software.amazon.kinesis.leases.LeaseRenewer; +import software.amazon.kinesis.leases.LeaseTaker; +import software.amazon.kinesis.leases.ShardInfo; +import software.amazon.kinesis.leases.exceptions.DependencyException; +import software.amazon.kinesis.leases.exceptions.InvalidStateException; +import software.amazon.kinesis.leases.exceptions.LeasingException; +import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException; +import software.amazon.kinesis.metrics.IMetricsFactory; +import software.amazon.kinesis.metrics.IMetricsScope; +import software.amazon.kinesis.metrics.MetricsHelper; +import software.amazon.kinesis.metrics.MetricsLevel; + +/** + * LeaseCoordinator abstracts away LeaseTaker and LeaseRenewer from the application code that's using leasing. It owns + * the scheduling of the two previously mentioned components as well as informing LeaseRenewer when LeaseTaker takes new + * leases. + * + */ +@Slf4j +public class DynamoDBLeaseCoordinator implements LeaseCoordinator { + /* + * Name of the dimension used when setting worker identifier on IMetricsScopes. Exposed so that users of this class + * can easily create InterceptingMetricsFactories that rename this dimension to suit the destination metrics system. + */ + private static final String WORKER_IDENTIFIER_METRIC = "WorkerIdentifier"; + // Time to wait for in-flight Runnables to finish when calling .stop(); + private static final long STOP_WAIT_TIME_MILLIS = 2000L; + private static final long DEFAULT_INITIAL_LEASE_TABLE_READ_CAPACITY = 10L; + private static final long DEFAULT_INITIAL_LEASE_TABLE_WRITE_CAPACITY = 10L; + private static final ThreadFactory LEASE_COORDINATOR_THREAD_FACTORY = new ThreadFactoryBuilder() + .setNameFormat("LeaseCoordinator-%04d").setDaemon(true).build(); + private static final ThreadFactory LEASE_RENEWAL_THREAD_FACTORY = new ThreadFactoryBuilder() + .setNameFormat("LeaseRenewer-%04d").setDaemon(true).build(); + + private final LeaseRenewer leaseRenewer; + private final LeaseTaker leaseTaker; + private final long renewerIntervalMillis; + private final long takerIntervalMillis; + private final ExecutorService leaseRenewalThreadpool; + private final LeaseRefresher leaseRefresher; + private final Object shutdownLock = new Object(); + protected final IMetricsFactory metricsFactory; + + private long initialLeaseTableReadCapacity = DEFAULT_INITIAL_LEASE_TABLE_READ_CAPACITY; + private long initialLeaseTableWriteCapacity = DEFAULT_INITIAL_LEASE_TABLE_WRITE_CAPACITY; + private ScheduledExecutorService leaseCoordinatorThreadPool; + private ScheduledFuture takerFuture; + + private volatile boolean running = false; + + /** + * Constructor. + * + * @param leaseRefresher LeaseRefresher instance to use + * @param workerIdentifier Identifies the worker (e.g. useful to track lease ownership) + * @param leaseDurationMillis Duration of a lease + * @param epsilonMillis Allow for some variance when calculating lease expirations + * @param maxLeasesForWorker Max leases this Worker can handle at a time + * @param maxLeasesToStealAtOneTime Steal up to these many leases at a time (for load balancing) + * @param metricsFactory Used to publish metrics about lease operations + */ + public DynamoDBLeaseCoordinator(final LeaseRefresher leaseRefresher, + final String workerIdentifier, + final long leaseDurationMillis, + final long epsilonMillis, + final int maxLeasesForWorker, + final int maxLeasesToStealAtOneTime, + final int maxLeaseRenewerThreadCount, + final IMetricsFactory metricsFactory) { + this.leaseRefresher = leaseRefresher; + this.leaseRenewalThreadpool = getLeaseRenewalExecutorService(maxLeaseRenewerThreadCount); + this.leaseTaker = new DynamoDBLeaseTaker(leaseRefresher, workerIdentifier, leaseDurationMillis) + .withMaxLeasesForWorker(maxLeasesForWorker) + .withMaxLeasesToStealAtOneTime(maxLeasesToStealAtOneTime); + this.leaseRenewer = new DynamoDBLeaseRenewer( + leaseRefresher, workerIdentifier, leaseDurationMillis, leaseRenewalThreadpool); + this.renewerIntervalMillis = leaseDurationMillis / 3 - epsilonMillis; + this.takerIntervalMillis = (leaseDurationMillis + epsilonMillis) * 2; + this.metricsFactory = metricsFactory; + + log.info("With failover time {} ms and epsilon {} ms, LeaseCoordinator will renew leases every {} ms, take" + + "leases every {} ms, process maximum of {} leases and steal {} lease(s) at a time.", + leaseDurationMillis, + epsilonMillis, + renewerIntervalMillis, + takerIntervalMillis, + maxLeasesForWorker, + maxLeasesToStealAtOneTime); + } + + private class TakerRunnable implements Runnable { + + @Override + public void run() { + try { + runLeaseTaker(); + } catch (LeasingException e) { + log.error("LeasingException encountered in lease taking thread", e); + } catch (Throwable t) { + log.error("Throwable encountered in lease taking thread", t); + } + } + + } + + private class RenewerRunnable implements Runnable { + + @Override + public void run() { + try { + runLeaseRenewer(); + } catch (LeasingException e) { + log.error("LeasingException encountered in lease renewing thread", e); + } catch (Throwable t) { + log.error("Throwable encountered in lease renewing thread", t); + } + } + + } + + @Override + public void initialize() throws ProvisionedThroughputException, DependencyException, IllegalStateException { + final boolean newTableCreated = + leaseRefresher.createLeaseTableIfNotExists(initialLeaseTableReadCapacity, initialLeaseTableWriteCapacity); + if (newTableCreated) { + log.info("Created new lease table for coordinator with initial read capacity of {} and write capacity of {}.", + initialLeaseTableReadCapacity, initialLeaseTableWriteCapacity); + } + // Need to wait for table in active state. + final long secondsBetweenPolls = 10L; + final long timeoutSeconds = 600L; + final boolean isTableActive = leaseRefresher.waitUntilLeaseTableExists(secondsBetweenPolls, timeoutSeconds); + if (!isTableActive) { + throw new DependencyException(new IllegalStateException("Creating table timeout")); + } + } + + @Override + public void start() throws DependencyException, InvalidStateException, ProvisionedThroughputException { + leaseRenewer.initialize(); + + // 2 because we know we'll have at most 2 concurrent tasks at a time. + leaseCoordinatorThreadPool = Executors.newScheduledThreadPool(2, LEASE_COORDINATOR_THREAD_FACTORY); + + // Taker runs with fixed DELAY because we want it to run slower in the event of performance degredation. + takerFuture = leaseCoordinatorThreadPool.scheduleWithFixedDelay(new TakerRunnable(), + 0L, + takerIntervalMillis, + TimeUnit.MILLISECONDS); + // Renewer runs at fixed INTERVAL because we want it to run at the same rate in the event of degredation. + leaseCoordinatorThreadPool.scheduleAtFixedRate(new RenewerRunnable(), + 0L, + renewerIntervalMillis, + TimeUnit.MILLISECONDS); + running = true; + } + + @Override + public void runLeaseTaker() throws DependencyException, InvalidStateException { + IMetricsScope scope = MetricsHelper.startScope(metricsFactory, "TakeLeases"); + long startTime = System.currentTimeMillis(); + boolean success = false; + + try { + Map takenLeases = leaseTaker.takeLeases(); + + // Only add taken leases to renewer if coordinator is still running. + synchronized (shutdownLock) { + if (running) { + leaseRenewer.addLeasesToRenew(takenLeases.values()); + } + } + + success = true; + } finally { + scope.addDimension(WORKER_IDENTIFIER_METRIC, workerIdentifier()); + MetricsHelper.addSuccessAndLatency(startTime, success, MetricsLevel.SUMMARY); + MetricsHelper.endScope(); + } + } + + @Override + public void runLeaseRenewer() throws DependencyException, InvalidStateException { + IMetricsScope scope = MetricsHelper.startScope(metricsFactory, "RenewAllLeases"); + long startTime = System.currentTimeMillis(); + boolean success = false; + + try { + leaseRenewer.renewLeases(); + success = true; + } finally { + scope.addDimension(WORKER_IDENTIFIER_METRIC, workerIdentifier()); + MetricsHelper.addSuccessAndLatency(startTime, success, MetricsLevel.SUMMARY); + MetricsHelper.endScope(); + } + } + + @Override + public Collection getAssignments() { + return leaseRenewer.getCurrentlyHeldLeases().values(); + } + + @Override + public Lease getCurrentlyHeldLease(String leaseKey) { + return leaseRenewer.getCurrentlyHeldLease(leaseKey); + } + + @Override + public String workerIdentifier() { + return leaseTaker.getWorkerIdentifier(); + } + + @Override + public LeaseRefresher leaseRefresher() { + return leaseRefresher; + } + + @Override + public void stop() { + if (leaseCoordinatorThreadPool != null) { + leaseCoordinatorThreadPool.shutdown(); + try { + if (leaseCoordinatorThreadPool.awaitTermination(STOP_WAIT_TIME_MILLIS, TimeUnit.MILLISECONDS)) { + log.info("Worker {} has successfully stopped lease-tracking threads", + leaseTaker.getWorkerIdentifier()); + } else { + leaseCoordinatorThreadPool.shutdownNow(); + log.info("Worker {} stopped lease-tracking threads {} ms after stop", + leaseTaker.getWorkerIdentifier(), + STOP_WAIT_TIME_MILLIS); + } + } catch (InterruptedException e) { + log.debug("Encountered InterruptedException when awaiting threadpool termination"); + } + } else { + log.debug("Threadpool was null, no need to shutdown/terminate threadpool."); + } + + leaseRenewalThreadpool.shutdownNow(); + synchronized (shutdownLock) { + leaseRenewer.clearCurrentlyHeldLeases(); + running = false; + } + } + + @Override + public void stopLeaseTaker() { + takerFuture.cancel(false); + + } + + @Override + public void dropLease(final Lease lease) { + synchronized (shutdownLock) { + if (lease != null) { + leaseRenewer.dropLease(lease); + } + } + } + + @Override + public boolean isRunning() { + return running; + } + + @Override + public boolean updateLease(final Lease lease, final UUID concurrencyToken) + throws DependencyException, InvalidStateException, ProvisionedThroughputException { + return leaseRenewer.updateLease(lease, concurrencyToken); + } + + /** + * Returns executor service that should be used for lease renewal. + * @param maximumPoolSize Maximum allowed thread pool size + * @return Executor service that should be used for lease renewal. + */ + private static ExecutorService getLeaseRenewalExecutorService(int maximumPoolSize) { + int coreLeaseCount = Math.max(maximumPoolSize / 4, 2); + + return new ThreadPoolExecutor(coreLeaseCount, maximumPoolSize, 60, TimeUnit.SECONDS, + new LinkedTransferQueue<>(), LEASE_RENEWAL_THREAD_FACTORY); + } + + @Override + public List getCurrentAssignments() { + Collection leases = getAssignments(); + return convertLeasesToAssignments(leases); + } + + private static List convertLeasesToAssignments(final Collection leases) { + if (CollectionUtils.isNullOrEmpty(leases)) { + return Collections.emptyList(); + } + return leases.stream().map(DynamoDBLeaseCoordinator::convertLeaseToAssignment).collect(Collectors.toList()); + } + + public static ShardInfo convertLeaseToAssignment(final Lease lease) { + return new ShardInfo(lease.leaseKey(), lease.concurrencyToken().toString(), lease.parentShardIds(), + lease.checkpoint()); + } + + @Override + public DynamoDBLeaseCoordinator initialLeaseTableReadCapacity(long readCapacity) { + if (readCapacity <= 0) { + throw new IllegalArgumentException("readCapacity should be >= 1"); + } + this.initialLeaseTableReadCapacity = readCapacity; + return this; + } + + @Override + public DynamoDBLeaseCoordinator initialLeaseTableWriteCapacity(long writeCapacity) { + if (writeCapacity <= 0) { + throw new IllegalArgumentException("writeCapacity should be >= 1"); + } + this.initialLeaseTableWriteCapacity = writeCapacity; + return this; + } +} diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/DynamoDBLeaseManagementFactory.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseManagementFactory.java similarity index 76% rename from amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/DynamoDBLeaseManagementFactory.java rename to amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseManagementFactory.java index 0518fd14..e9945dea 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/DynamoDBLeaseManagementFactory.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseManagementFactory.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package software.amazon.kinesis.leases; +package software.amazon.kinesis.leases.dynamodb; import java.util.concurrent.ExecutorService; @@ -23,6 +23,11 @@ import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionIn import lombok.Data; import lombok.NonNull; +import software.amazon.kinesis.leases.KinesisShardDetector; +import software.amazon.kinesis.leases.LeaseCoordinator; +import software.amazon.kinesis.leases.LeaseManagementFactory; +import software.amazon.kinesis.leases.ShardDetector; +import software.amazon.kinesis.leases.ShardSyncTaskManager; import software.amazon.kinesis.metrics.IMetricsFactory; /** @@ -60,29 +65,7 @@ public class DynamoDBLeaseManagementFactory implements LeaseManagementFactory { @Override public LeaseCoordinator createLeaseCoordinator() { - return createKinesisClientLibLeaseCoordinator(); - } - - @Override - public ShardSyncTaskManager createShardSyncTaskManager() { - return new ShardSyncTaskManager(this.createLeaseManagerProxy(), - this.createLeaseManager(), - initialPositionInStream, - cleanupLeasesUponShardCompletion, - ignoreUnexpectedChildShards, - shardSyncIntervalMillis, - executorService, - metricsFactory); - } - - @Override - public DynamoDBLeaseManager createLeaseManager() { - return new KinesisClientDynamoDBLeaseManager(tableName, amazonDynamoDB, consistentReads); - } - - @Override - public KinesisClientLibLeaseCoordinator createKinesisClientLibLeaseCoordinator() { - return new KinesisClientLibLeaseCoordinator(this.createLeaseManager(), + return new DynamoDBLeaseCoordinator(this.createLeaseRefresher(), workerIdentifier, failoverTimeMillis, epsilonMillis, @@ -93,8 +76,25 @@ public class DynamoDBLeaseManagementFactory implements LeaseManagementFactory { } @Override - public LeaseManagerProxy createLeaseManagerProxy() { - return new KinesisLeaseManagerProxy(amazonKinesis, streamName, listShardsBackoffTimeMillis, + public ShardSyncTaskManager createShardSyncTaskManager() { + return new ShardSyncTaskManager(this.createShardDetector(), + this.createLeaseRefresher(), + initialPositionInStream, + cleanupLeasesUponShardCompletion, + ignoreUnexpectedChildShards, + shardSyncIntervalMillis, + executorService, + metricsFactory); + } + + @Override + public DynamoDBLeaseRefresher createLeaseRefresher() { + return new DynamoDBLeaseRefresher(tableName, amazonDynamoDB, new DynamoDBLeaseSerializer(), consistentReads); + } + + @Override + public ShardDetector createShardDetector() { + return new KinesisShardDetector(amazonKinesis, streamName, listShardsBackoffTimeMillis, maxListShardsRetryAttempts); } } diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/DynamoDBLeaseManager.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseRefresher.java similarity index 75% rename from amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/DynamoDBLeaseManager.java rename to amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseRefresher.java index 49dec77f..7882b23d 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/DynamoDBLeaseManager.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseRefresher.java @@ -12,7 +12,7 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package software.amazon.kinesis.leases; +package software.amazon.kinesis.leases.dynamodb; import java.util.ArrayList; import java.util.List; @@ -40,33 +40,27 @@ import com.amazonaws.services.dynamodbv2.model.ScanRequest; import com.amazonaws.services.dynamodbv2.model.ScanResult; import com.amazonaws.services.dynamodbv2.model.TableStatus; import com.amazonaws.services.dynamodbv2.model.UpdateItemRequest; + +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import software.amazon.kinesis.leases.Lease; +import software.amazon.kinesis.leases.LeaseRefresher; +import software.amazon.kinesis.leases.LeaseSerializer; import software.amazon.kinesis.leases.exceptions.DependencyException; import software.amazon.kinesis.leases.exceptions.InvalidStateException; import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException; - -import lombok.extern.slf4j.Slf4j; +import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; /** - * An implementation of ILeaseManager that uses DynamoDB. + * An implementation of {@link LeaseRefresher} that uses DynamoDB. */ @Slf4j -public class DynamoDBLeaseManager implements LeaseManager { +public class DynamoDBLeaseRefresher implements LeaseRefresher { protected String table; protected AmazonDynamoDB dynamoDBClient; - protected LeaseSerializer serializer; + protected LeaseSerializer serializer; protected boolean consistentReads; - /** - * Constructor. - * - * @param table leases table - * @param dynamoDBClient DynamoDB client to use - * @param serializer LeaseSerializer to use to convert to/from DynamoDB objects. - */ - public DynamoDBLeaseManager(String table, AmazonDynamoDB dynamoDBClient, LeaseSerializer serializer) { - this(table, dynamoDBClient, serializer, false); - } - /** * Constructor for test cases - allows control of consistent reads. Consistent reads should only be used for testing * - our code is meant to be resilient to inconsistent reads. Using consistent reads during testing speeds up @@ -78,11 +72,8 @@ public class DynamoDBLeaseManager implements LeaseManager { * @param serializer lease serializer to use * @param consistentReads true if we want consistent reads for testing purposes. */ - public DynamoDBLeaseManager(String table, AmazonDynamoDB dynamoDBClient, LeaseSerializer serializer, boolean consistentReads) { - verifyNotNull(table, "Table name cannot be null"); - verifyNotNull(dynamoDBClient, "dynamoDBClient cannot be null"); - verifyNotNull(serializer, "ILeaseSerializer cannot be null"); - + public DynamoDBLeaseRefresher(@NonNull final String table, @NonNull final AmazonDynamoDB dynamoDBClient, + @NonNull final LeaseSerializer serializer, boolean consistentReads) { this.table = table; this.dynamoDBClient = dynamoDBClient; this.consistentReads = consistentReads; @@ -93,11 +84,8 @@ public class DynamoDBLeaseManager implements LeaseManager { * {@inheritDoc} */ @Override - public boolean createLeaseTableIfNotExists(Long readCapacity, Long writeCapacity) - throws ProvisionedThroughputException, DependencyException { - verifyNotNull(readCapacity, "readCapacity cannot be null"); - verifyNotNull(writeCapacity, "writeCapacity cannot be null"); - + public boolean createLeaseTableIfNotExists(@NonNull final Long readCapacity, @NonNull final Long writeCapacity) + throws ProvisionedThroughputException, DependencyException { try { if (tableStatus() != null) { return false; @@ -204,7 +192,7 @@ public class DynamoDBLeaseManager implements LeaseManager { * {@inheritDoc} */ @Override - public List listLeases() throws DependencyException, InvalidStateException, ProvisionedThroughputException { + public List listLeases() throws DependencyException, InvalidStateException, ProvisionedThroughputException { return list(null); } @@ -212,7 +200,8 @@ public class DynamoDBLeaseManager implements LeaseManager { * {@inheritDoc} */ @Override - public boolean isLeaseTableEmpty() throws DependencyException, InvalidStateException, ProvisionedThroughputException { + public boolean isLeaseTableEmpty() + throws DependencyException, InvalidStateException, ProvisionedThroughputException { return list(1).isEmpty(); } @@ -225,7 +214,7 @@ public class DynamoDBLeaseManager implements LeaseManager { * @throws DependencyException if DynamoDB scan fail in an unexpected way * @throws ProvisionedThroughputException if DynamoDB scan fail due to exceeded capacity */ - List list(Integer limit) throws DependencyException, InvalidStateException, ProvisionedThroughputException { + List list(Integer limit) throws DependencyException, InvalidStateException, ProvisionedThroughputException { if (log.isDebugEnabled()) { log.debug("Listing leases from table {}", table); } @@ -238,7 +227,7 @@ public class DynamoDBLeaseManager implements LeaseManager { try { ScanResult scanResult = dynamoDBClient.scan(scanRequest); - List result = new ArrayList(); + List result = new ArrayList<>(); while (scanResult != null) { for (Map item : scanResult.getItems()) { @@ -286,10 +275,8 @@ public class DynamoDBLeaseManager implements LeaseManager { * {@inheritDoc} */ @Override - public boolean createLeaseIfNotExists(T lease) - throws DependencyException, InvalidStateException, ProvisionedThroughputException { - verifyNotNull(lease, "lease cannot be null"); - + public boolean createLeaseIfNotExists(@NonNull final Lease lease) + throws DependencyException, InvalidStateException, ProvisionedThroughputException { if (log.isDebugEnabled()) { log.debug("Creating lease {}", lease); } @@ -308,7 +295,7 @@ public class DynamoDBLeaseManager implements LeaseManager { return false; } catch (AmazonClientException e) { - throw convertAndRethrowExceptions("create", lease.getLeaseKey(), e); + throw convertAndRethrowExceptions("create", lease.leaseKey(), e); } return true; @@ -318,10 +305,8 @@ public class DynamoDBLeaseManager implements LeaseManager { * {@inheritDoc} */ @Override - public T getLease(String leaseKey) - throws DependencyException, InvalidStateException, ProvisionedThroughputException { - verifyNotNull(leaseKey, "leaseKey cannot be null"); - + public Lease getLease(@NonNull final String leaseKey) + throws DependencyException, InvalidStateException, ProvisionedThroughputException { if (log.isDebugEnabled()) { log.debug("Getting lease with key {}", leaseKey); } @@ -342,7 +327,7 @@ public class DynamoDBLeaseManager implements LeaseManager { return null; } else { - T lease = serializer.fromDynamoRecord(dynamoRecord); + final Lease lease = serializer.fromDynamoRecord(dynamoRecord); if (log.isDebugEnabled()) { log.debug("Got lease {}", lease); } @@ -358,12 +343,10 @@ public class DynamoDBLeaseManager implements LeaseManager { * {@inheritDoc} */ @Override - public boolean renewLease(T lease) - throws DependencyException, InvalidStateException, ProvisionedThroughputException { - verifyNotNull(lease, "lease cannot be null"); - + public boolean renewLease(@NonNull final Lease lease) + throws DependencyException, InvalidStateException, ProvisionedThroughputException { if (log.isDebugEnabled()) { - log.debug("Renewing lease with key {}", lease.getLeaseKey()); + log.debug("Renewing lease with key {}", lease.leaseKey()); } UpdateItemRequest request = new UpdateItemRequest(); @@ -377,26 +360,26 @@ public class DynamoDBLeaseManager implements LeaseManager { } catch (ConditionalCheckFailedException e) { if (log.isDebugEnabled()) { log.debug("Lease renewal failed for lease with key {} because the lease counter was not {}", - lease.getLeaseKey(), lease.getLeaseCounter()); + lease.leaseKey(), lease.leaseCounter()); } // If we had a spurious retry during the Dynamo update, then this conditional PUT failure // might be incorrect. So, we get the item straight away and check if the lease owner + lease counter // are what we expected. - String expectedOwner = lease.getLeaseOwner(); - Long expectedCounter = lease.getLeaseCounter() + 1; - T updatedLease = getLease(lease.getLeaseKey()); - if (updatedLease == null || !expectedOwner.equals(updatedLease.getLeaseOwner()) || - !expectedCounter.equals(updatedLease.getLeaseCounter())) { + String expectedOwner = lease.leaseOwner(); + Long expectedCounter = lease.leaseCounter() + 1; + final Lease updatedLease = getLease(lease.leaseKey()); + if (updatedLease == null || !expectedOwner.equals(updatedLease.leaseOwner()) || + !expectedCounter.equals(updatedLease.leaseCounter())) { return false; } - log.info("Detected spurious renewal failure for lease with key {}, but recovered", lease.getLeaseKey()); + log.info("Detected spurious renewal failure for lease with key {}, but recovered", lease.leaseKey()); } catch (AmazonClientException e) { - throw convertAndRethrowExceptions("renew", lease.getLeaseKey(), e); + throw convertAndRethrowExceptions("renew", lease.leaseKey(), e); } - lease.setLeaseCounter(lease.getLeaseCounter() + 1); + lease.leaseCounter(lease.leaseCounter() + 1); return true; } @@ -404,15 +387,14 @@ public class DynamoDBLeaseManager implements LeaseManager { * {@inheritDoc} */ @Override - public boolean takeLease(T lease, String owner) - throws DependencyException, InvalidStateException, ProvisionedThroughputException { - verifyNotNull(lease, "lease cannot be null"); - verifyNotNull(owner, "owner cannot be null"); + public boolean takeLease(@NonNull final Lease lease, @NonNull final String owner) + throws DependencyException, InvalidStateException, ProvisionedThroughputException { + final String oldOwner = lease.leaseOwner(); if (log.isDebugEnabled()) { log.debug("Taking lease with leaseKey {} from {} to {}", - lease.getLeaseKey(), - lease.getLeaseOwner() == null ? "nobody" : lease.getLeaseOwner(), + lease.leaseKey(), + lease.leaseOwner() == null ? "nobody" : lease.leaseOwner(), owner); } @@ -430,16 +412,20 @@ public class DynamoDBLeaseManager implements LeaseManager { } catch (ConditionalCheckFailedException e) { if (log.isDebugEnabled()) { log.debug("Lease renewal failed for lease with key {} because the lease counter was not {}", - lease.getLeaseKey(), lease.getLeaseCounter()); + lease.leaseKey(), lease.leaseCounter()); } return false; } catch (AmazonClientException e) { - throw convertAndRethrowExceptions("take", lease.getLeaseKey(), e); + throw convertAndRethrowExceptions("take", lease.leaseKey(), e); } - lease.setLeaseCounter(lease.getLeaseCounter() + 1); - lease.setLeaseOwner(owner); + lease.leaseCounter(lease.leaseCounter() + 1); + lease.leaseOwner(owner); + + if (oldOwner != null && !oldOwner.equals(owner)) { + lease.ownerSwitchesSinceCheckpoint(lease.ownerSwitchesSinceCheckpoint() + 1); + } return true; } @@ -448,14 +434,10 @@ public class DynamoDBLeaseManager implements LeaseManager { * {@inheritDoc} */ @Override - public boolean evictLease(T lease) - throws DependencyException, InvalidStateException, ProvisionedThroughputException { - verifyNotNull(lease, "lease cannot be null"); - + public boolean evictLease(@NonNull final Lease lease) + throws DependencyException, InvalidStateException, ProvisionedThroughputException { if (log.isDebugEnabled()) { - log.debug("Evicting lease with leaseKey {} owned by {}", - lease.getLeaseKey(), - lease.getLeaseOwner()); + log.debug("Evicting lease with leaseKey {} owned by {}", lease.leaseKey(), lease.leaseOwner()); } UpdateItemRequest request = new UpdateItemRequest(); @@ -472,16 +454,16 @@ public class DynamoDBLeaseManager implements LeaseManager { } catch (ConditionalCheckFailedException e) { if (log.isDebugEnabled()) { log.debug("Lease eviction failed for lease with key {} because the lease owner was not {}", - lease.getLeaseKey(), lease.getLeaseOwner()); + lease.leaseKey(), lease.leaseOwner()); } return false; } catch (AmazonClientException e) { - throw convertAndRethrowExceptions("evict", lease.getLeaseKey(), e); + throw convertAndRethrowExceptions("evict", lease.leaseKey(), e); } - lease.setLeaseOwner(null); - lease.setLeaseCounter(lease.getLeaseCounter() + 1); + lease.leaseOwner(null); + lease.leaseCounter(lease.leaseCounter() + 1); return true; } @@ -489,11 +471,11 @@ public class DynamoDBLeaseManager implements LeaseManager { * {@inheritDoc} */ public void deleteAll() throws DependencyException, InvalidStateException, ProvisionedThroughputException { - List allLeases = listLeases(); + List allLeases = listLeases(); log.warn("Deleting {} items from table {}", allLeases.size(), table); - for (T lease : allLeases) { + for (final Lease lease : allLeases) { DeleteItemRequest deleteRequest = new DeleteItemRequest(); deleteRequest.setTableName(table); deleteRequest.setKey(serializer.getDynamoHashKey(lease)); @@ -506,11 +488,10 @@ public class DynamoDBLeaseManager implements LeaseManager { * {@inheritDoc} */ @Override - public void deleteLease(T lease) throws DependencyException, InvalidStateException, ProvisionedThroughputException { - verifyNotNull(lease, "lease cannot be null"); - + public void deleteLease(@NonNull final Lease lease) + throws DependencyException, InvalidStateException, ProvisionedThroughputException { if (log.isDebugEnabled()) { - log.debug("Deleting lease with leaseKey {}", lease.getLeaseKey()); + log.debug("Deleting lease with leaseKey {}", lease.leaseKey()); } DeleteItemRequest deleteRequest = new DeleteItemRequest(); @@ -520,7 +501,7 @@ public class DynamoDBLeaseManager implements LeaseManager { try { dynamoDBClient.deleteItem(deleteRequest); } catch (AmazonClientException e) { - throw convertAndRethrowExceptions("delete", lease.getLeaseKey(), e); + throw convertAndRethrowExceptions("delete", lease.leaseKey(), e); } } @@ -528,10 +509,8 @@ public class DynamoDBLeaseManager implements LeaseManager { * {@inheritDoc} */ @Override - public boolean updateLease(T lease) - throws DependencyException, InvalidStateException, ProvisionedThroughputException { - verifyNotNull(lease, "lease cannot be null"); - + public boolean updateLease(@NonNull final Lease lease) + throws DependencyException, InvalidStateException, ProvisionedThroughputException { if (log.isDebugEnabled()) { log.debug("Updating lease {}", lease); } @@ -550,44 +529,53 @@ public class DynamoDBLeaseManager implements LeaseManager { } catch (ConditionalCheckFailedException e) { if (log.isDebugEnabled()) { log.debug("Lease update failed for lease with key {} because the lease counter was not {}", - lease.getLeaseKey(), lease.getLeaseCounter()); + lease.leaseKey(), lease.leaseCounter()); } return false; } catch (AmazonClientException e) { - throw convertAndRethrowExceptions("update", lease.getLeaseKey(), e); + throw convertAndRethrowExceptions("update", lease.leaseKey(), e); } - lease.setLeaseCounter(lease.getLeaseCounter() + 1); + lease.leaseCounter(lease.leaseCounter() + 1); return true; } + /** + * {@inheritDoc} + */ + @Override + public ExtendedSequenceNumber getCheckpoint(String shardId) + throws ProvisionedThroughputException, InvalidStateException, DependencyException { + ExtendedSequenceNumber checkpoint = null; + Lease lease = getLease(shardId); + if (lease != null) { + checkpoint = lease.checkpoint(); + } + return checkpoint; + } + /* * This method contains boilerplate exception handling - it throws or returns something to be thrown. The * inconsistency there exists to satisfy the compiler when this method is used at the end of non-void methods. */ - protected DependencyException convertAndRethrowExceptions(String operation, String leaseKey, AmazonClientException e) + protected DependencyException convertAndRethrowExceptions(String operation, String leaseKey, + AmazonClientException e) throws ProvisionedThroughputException, InvalidStateException { if (e instanceof ProvisionedThroughputExceededException) { - log.warn("Provisioned Throughput on the lease table has been exceeded. It's recommended that you increase the IOPs on the table. Failure to increase the IOPs may cause the application to not make progress."); + log.warn("Provisioned Throughput on the lease table has been exceeded. It's recommended that you increase" + + " the IOPs on the table. Failure to increase the IOPs may cause the application to not make" + + " progress."); throw new ProvisionedThroughputException(e); } else if (e instanceof ResourceNotFoundException) { // @formatter:on - throw new InvalidStateException(String.format("Cannot %s lease with key %s because table %s does not exist.", - operation, - leaseKey, - table), + throw new InvalidStateException( + String.format("Cannot %s lease with key %s because table %s does not exist.", operation, leaseKey, + table), e); //@formatter:off } else { return new DependencyException(e); } } - - private void verifyNotNull(Object object, String message) { - if (object == null) { - throw new IllegalArgumentException(message); - } - } - } diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/DynamoDBLeaseRenewer.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseRenewer.java similarity index 79% rename from amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/DynamoDBLeaseRenewer.java rename to amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseRenewer.java index 6ebd39a9..dd0dbd07 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/DynamoDBLeaseRenewer.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseRenewer.java @@ -12,7 +12,7 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package software.amazon.kinesis.leases; +package software.amazon.kinesis.leases.dynamodb; import java.util.ArrayList; import java.util.Collection; @@ -30,25 +30,28 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import com.amazonaws.services.cloudwatch.model.StandardUnit; + +import lombok.extern.slf4j.Slf4j; +import software.amazon.kinesis.leases.Lease; +import software.amazon.kinesis.leases.LeaseRefresher; +import software.amazon.kinesis.leases.LeaseRenewer; import software.amazon.kinesis.leases.exceptions.DependencyException; import software.amazon.kinesis.leases.exceptions.InvalidStateException; import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException; -import software.amazon.kinesis.metrics.MetricsHelper; -import software.amazon.kinesis.metrics.ThreadSafeMetricsDelegatingScope; import software.amazon.kinesis.metrics.IMetricsScope; +import software.amazon.kinesis.metrics.MetricsHelper; import software.amazon.kinesis.metrics.MetricsLevel; - -import lombok.extern.slf4j.Slf4j; +import software.amazon.kinesis.metrics.ThreadSafeMetricsDelegatingScope; /** - * An implementation of ILeaseRenewer that uses DynamoDB via LeaseManager. + * An implementation of {@link LeaseRenewer} that uses DynamoDB via {@link LeaseRefresher}. */ @Slf4j -public class DynamoDBLeaseRenewer implements LeaseRenewer { +public class DynamoDBLeaseRenewer implements LeaseRenewer { private static final int RENEWAL_RETRIES = 2; - private final LeaseManager leaseManager; - private final ConcurrentNavigableMap ownedLeases = new ConcurrentSkipListMap(); + private final LeaseRefresher leaseRefresher; + private final ConcurrentNavigableMap ownedLeases = new ConcurrentSkipListMap<>(); private final String workerIdentifier; private final long leaseDurationNanos; private final ExecutorService executorService; @@ -56,14 +59,14 @@ public class DynamoDBLeaseRenewer implements LeaseRenewer { /** * Constructor. * - * @param leaseManager LeaseManager to use + * @param leaseRefresher LeaseRefresher to use * @param workerIdentifier identifier of this worker * @param leaseDurationMillis duration of a lease in milliseconds * @param executorService ExecutorService to use for renewing leases in parallel */ - public DynamoDBLeaseRenewer(LeaseManager leaseManager, String workerIdentifier, long leaseDurationMillis, - ExecutorService executorService) { - this.leaseManager = leaseManager; + public DynamoDBLeaseRenewer(final LeaseRefresher leaseRefresher, final String workerIdentifier, + final long leaseDurationMillis, final ExecutorService executorService) { + this.leaseRefresher = leaseRefresher; this.workerIdentifier = workerIdentifier; this.leaseDurationNanos = TimeUnit.MILLISECONDS.toNanos(leaseDurationMillis); this.executorService = executorService; @@ -77,10 +80,7 @@ public class DynamoDBLeaseRenewer implements LeaseRenewer { if (log.isDebugEnabled()) { // Due to the eventually consistent nature of ConcurrentNavigableMap iterators, this log entry may become // inaccurate during iteration. - log.debug("Worker {} holding %d leases: {}", - workerIdentifier, - ownedLeases.size(), - ownedLeases); + log.debug("Worker {} holding {} leases: {}", workerIdentifier, ownedLeases.size(), ownedLeases); } /* @@ -96,8 +96,8 @@ public class DynamoDBLeaseRenewer implements LeaseRenewer { * to getCurrentlyHeldLeases. They'll still cross paths, but they won't interleave their executions. */ int lostLeases = 0; - List> renewLeaseTasks = new ArrayList>(); - for (T lease : ownedLeases.descendingMap().values()) { + List> renewLeaseTasks = new ArrayList<>(); + for (Lease lease : ownedLeases.descendingMap().values()) { renewLeaseTasks.add(executorService.submit(new RenewLeaseTask(lease, renewLeaseTaskMetricsScope))); } int leasesInUnknownState = 0; @@ -132,10 +132,10 @@ public class DynamoDBLeaseRenewer implements LeaseRenewer { private class RenewLeaseTask implements Callable { - private final T lease; + private final Lease lease; private final IMetricsScope metricsScope; - public RenewLeaseTask(T lease, IMetricsScope metricsScope) { + public RenewLeaseTask(Lease lease, IMetricsScope metricsScope) { this.lease = lease; this.metricsScope = metricsScope; } @@ -151,12 +151,12 @@ public class DynamoDBLeaseRenewer implements LeaseRenewer { } } - private boolean renewLease(T lease) throws DependencyException, InvalidStateException { + private boolean renewLease(Lease lease) throws DependencyException, InvalidStateException { return renewLease(lease, false); } - private boolean renewLease(T lease, boolean renewEvenIfExpired) throws DependencyException, InvalidStateException { - String leaseKey = lease.getLeaseKey(); + private boolean renewLease(Lease lease, boolean renewEvenIfExpired) throws DependencyException, InvalidStateException { + String leaseKey = lease.leaseKey(); boolean success = false; boolean renewedLease = false; @@ -170,10 +170,10 @@ public class DynamoDBLeaseRenewer implements LeaseRenewer { // ShutdownException). boolean isLeaseExpired = lease.isExpired(leaseDurationNanos, System.nanoTime()); if (renewEvenIfExpired || !isLeaseExpired) { - renewedLease = leaseManager.renewLease(lease); + renewedLease = leaseRefresher.renewLease(lease); } if (renewedLease) { - lease.setLastCounterIncrementNanos(System.nanoTime()); + lease.lastCounterIncrementNanos(System.nanoTime()); } } @@ -209,14 +209,14 @@ public class DynamoDBLeaseRenewer implements LeaseRenewer { * {@inheritDoc} */ @Override - public Map getCurrentlyHeldLeases() { - Map result = new HashMap(); + public Map getCurrentlyHeldLeases() { + Map result = new HashMap<>(); long now = System.nanoTime(); for (String leaseKey : ownedLeases.keySet()) { - T copy = getCopyOfHeldLease(leaseKey, now); + Lease copy = getCopyOfHeldLease(leaseKey, now); if (copy != null) { - result.put(copy.getLeaseKey(), copy); + result.put(copy.leaseKey(), copy); } } @@ -227,7 +227,7 @@ public class DynamoDBLeaseRenewer implements LeaseRenewer { * {@inheritDoc} */ @Override - public T getCurrentlyHeldLease(String leaseKey) { + public Lease getCurrentlyHeldLease(String leaseKey) { return getCopyOfHeldLease(leaseKey, System.nanoTime()); } @@ -238,19 +238,19 @@ public class DynamoDBLeaseRenewer implements LeaseRenewer { * @param now current timestamp for old-ness checking * @return non-authoritative copy of the held lease, or null if we don't currently hold it */ - private T getCopyOfHeldLease(String leaseKey, long now) { - T authoritativeLease = ownedLeases.get(leaseKey); + private Lease getCopyOfHeldLease(String leaseKey, long now) { + Lease authoritativeLease = ownedLeases.get(leaseKey); if (authoritativeLease == null) { return null; } else { - T copy = null; + Lease copy = null; synchronized (authoritativeLease) { copy = authoritativeLease.copy(); } if (copy.isExpired(leaseDurationNanos, now)) { log.info("getCurrentlyHeldLease not returning lease with key {} because it is expired", - copy.getLeaseKey()); + copy.leaseKey()); return null; } else { return copy; @@ -262,14 +262,14 @@ public class DynamoDBLeaseRenewer implements LeaseRenewer { * {@inheritDoc} */ @Override - public boolean updateLease(T lease, UUID concurrencyToken) + public boolean updateLease(Lease lease, UUID concurrencyToken) throws DependencyException, InvalidStateException, ProvisionedThroughputException { verifyNotNull(lease, "lease cannot be null"); - verifyNotNull(lease.getLeaseKey(), "leaseKey cannot be null"); + verifyNotNull(lease.leaseKey(), "leaseKey cannot be null"); verifyNotNull(concurrencyToken, "concurrencyToken cannot be null"); - String leaseKey = lease.getLeaseKey(); - T authoritativeLease = ownedLeases.get(leaseKey); + String leaseKey = lease.leaseKey(); + Lease authoritativeLease = ownedLeases.get(leaseKey); if (authoritativeLease == null) { log.info("Worker {} could not update lease with key {} because it does not hold it", @@ -283,7 +283,7 @@ public class DynamoDBLeaseRenewer implements LeaseRenewer { * the lease was lost and regained between when the caller acquired his concurrency token and when the caller * called update. */ - if (!authoritativeLease.getConcurrencyToken().equals(concurrencyToken)) { + if (!authoritativeLease.concurrencyToken().equals(concurrencyToken)) { log.info("Worker {} refusing to update lease with key {} because" + " concurrency tokens don't match", workerIdentifier, leaseKey); return false; @@ -294,10 +294,10 @@ public class DynamoDBLeaseRenewer implements LeaseRenewer { try { synchronized (authoritativeLease) { authoritativeLease.update(lease); - boolean updatedLease = leaseManager.updateLease(authoritativeLease); + boolean updatedLease = leaseRefresher.updateLease(authoritativeLease); if (updatedLease) { // Updates increment the counter - authoritativeLease.setLastCounterIncrementNanos(System.nanoTime()); + authoritativeLease.lastCounterIncrementNanos(System.nanoTime()); } else { /* * If updateLease returns false, it means someone took the lease from us. Remove the lease @@ -313,7 +313,7 @@ public class DynamoDBLeaseRenewer implements LeaseRenewer { * * 1) Concurrency token check passes * 2) Pause. Lose lease, re-acquire lease. This requires at least one lease counter update. - * 3) Unpause. leaseManager.updateLease fails conditional write due to counter updates, returns + * 3) Unpause. leaseRefresher.updateLease fails conditional write due to counter updates, returns * false. * 4) ownedLeases.remove(key, value) doesn't do anything because authoritativeLease does not * .equals() the re-acquired version in the map on the basis of lease counter. This is what we want. @@ -337,24 +337,24 @@ public class DynamoDBLeaseRenewer implements LeaseRenewer { * {@inheritDoc} */ @Override - public void addLeasesToRenew(Collection newLeases) { + public void addLeasesToRenew(Collection newLeases) { verifyNotNull(newLeases, "newLeases cannot be null"); - for (T lease : newLeases) { - if (lease.getLastCounterIncrementNanos() == null) { + for (Lease lease : newLeases) { + if (lease.lastCounterIncrementNanos() == null) { log.info("addLeasesToRenew ignoring lease with key {} because it does not have lastRenewalNanos set", - lease.getLeaseKey()); + lease.leaseKey()); continue; } - T authoritativeLease = lease.copy(); + Lease authoritativeLease = lease.copy(); /* * Assign a concurrency token when we add this to the set of currently owned leases. This ensures that * every time we acquire a lease, it gets a new concurrency token. */ - authoritativeLease.setConcurrencyToken(UUID.randomUUID()); - ownedLeases.put(authoritativeLease.getLeaseKey(), authoritativeLease); + authoritativeLease.concurrencyToken(UUID.randomUUID()); + ownedLeases.put(authoritativeLease.leaseKey(), authoritativeLease); } } @@ -371,8 +371,8 @@ public class DynamoDBLeaseRenewer implements LeaseRenewer { * @param lease the lease to drop. */ @Override - public void dropLease(T lease) { - ownedLeases.remove(lease.getLeaseKey()); + public void dropLease(Lease lease) { + ownedLeases.remove(lease.leaseKey()); } /** @@ -380,12 +380,12 @@ public class DynamoDBLeaseRenewer implements LeaseRenewer { */ @Override public void initialize() throws DependencyException, InvalidStateException, ProvisionedThroughputException { - Collection leases = leaseManager.listLeases(); - List myLeases = new LinkedList(); + Collection leases = leaseRefresher.listLeases(); + List myLeases = new LinkedList<>(); boolean renewEvenIfExpired = true; - for (T lease : leases) { - if (workerIdentifier.equals(lease.getLeaseOwner())) { + for (Lease lease : leases) { + if (workerIdentifier.equals(lease.leaseOwner())) { log.info(" Worker {} found lease {}", workerIdentifier, lease); // Okay to renew even if lease is expired, because we start with an empty list and we add the lease to // our list only after a successful renew. So we don't need to worry about the edge case where we could diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseSerializer.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseSerializer.java new file mode 100644 index 00000000..642bd4ad --- /dev/null +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseSerializer.java @@ -0,0 +1,236 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package software.amazon.kinesis.leases.dynamodb; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.amazonaws.services.dynamodbv2.model.AttributeAction; +import com.amazonaws.services.dynamodbv2.model.AttributeDefinition; +import com.amazonaws.services.dynamodbv2.model.AttributeValue; +import com.amazonaws.services.dynamodbv2.model.AttributeValueUpdate; +import com.amazonaws.services.dynamodbv2.model.ExpectedAttributeValue; +import com.amazonaws.services.dynamodbv2.model.KeySchemaElement; +import com.amazonaws.services.dynamodbv2.model.KeyType; +import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType; +import com.google.common.base.Strings; +import software.amazon.kinesis.leases.DynamoUtils; +import software.amazon.kinesis.leases.Lease; +import software.amazon.kinesis.leases.LeaseSerializer; +import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; + +/** + * An implementation of ILeaseSerializer for basic Lease objects. Can also instantiate subclasses of Lease so that + * LeaseSerializer can be decorated by other classes if you need to add fields to leases. + */ +public class DynamoDBLeaseSerializer implements LeaseSerializer { + private static final String LEASE_KEY_KEY = "leaseKey"; + private static final String LEASE_OWNER_KEY = "leaseOwner"; + private static final String LEASE_COUNTER_KEY = "leaseCounter"; + private static final String OWNER_SWITCHES_KEY = "ownerSwitchesSinceCheckpoint"; + private static final String CHECKPOINT_SEQUENCE_NUMBER_KEY = "checkpoint"; + private static final String CHECKPOINT_SUBSEQUENCE_NUMBER_KEY = "checkpointSubSequenceNumber"; + private static final String PENDING_CHECKPOINT_SEQUENCE_KEY = "pendingCheckpoint"; + private static final String PENDING_CHECKPOINT_SUBSEQUENCE_KEY = "pendingCheckpointSubSequenceNumber"; + private static final String PARENT_SHARD_ID_KEY = "parentShardId"; + + @Override + public Map toDynamoRecord(final Lease lease) { + Map result = new HashMap<>(); + + result.put(LEASE_KEY_KEY, DynamoUtils.createAttributeValue(lease.leaseKey())); + result.put(LEASE_COUNTER_KEY, DynamoUtils.createAttributeValue(lease.leaseCounter())); + + if (lease.leaseOwner() != null) { + result.put(LEASE_OWNER_KEY, DynamoUtils.createAttributeValue(lease.leaseOwner())); + } + + result.put(OWNER_SWITCHES_KEY, DynamoUtils.createAttributeValue(lease.ownerSwitchesSinceCheckpoint())); + result.put(CHECKPOINT_SEQUENCE_NUMBER_KEY, DynamoUtils.createAttributeValue(lease.checkpoint().getSequenceNumber())); + result.put(CHECKPOINT_SUBSEQUENCE_NUMBER_KEY, DynamoUtils.createAttributeValue(lease.checkpoint().getSubSequenceNumber())); + if (lease.parentShardIds() != null && !lease.parentShardIds().isEmpty()) { + result.put(PARENT_SHARD_ID_KEY, DynamoUtils.createAttributeValue(lease.parentShardIds())); + } + + if (lease.pendingCheckpoint() != null && !lease.pendingCheckpoint().getSequenceNumber().isEmpty()) { + result.put(PENDING_CHECKPOINT_SEQUENCE_KEY, DynamoUtils.createAttributeValue(lease.pendingCheckpoint().getSequenceNumber())); + result.put(PENDING_CHECKPOINT_SUBSEQUENCE_KEY, DynamoUtils.createAttributeValue(lease.pendingCheckpoint().getSubSequenceNumber())); + } + + return result; + } + + @Override + public Lease fromDynamoRecord(final Map dynamoRecord) { + Lease result = new Lease(); + result.leaseKey(DynamoUtils.safeGetString(dynamoRecord, LEASE_KEY_KEY)); + result.leaseOwner(DynamoUtils.safeGetString(dynamoRecord, LEASE_OWNER_KEY)); + result.leaseCounter(DynamoUtils.safeGetLong(dynamoRecord, LEASE_COUNTER_KEY)); + + result.ownerSwitchesSinceCheckpoint(DynamoUtils.safeGetLong(dynamoRecord, OWNER_SWITCHES_KEY)); + result.checkpoint( + new ExtendedSequenceNumber( + DynamoUtils.safeGetString(dynamoRecord, CHECKPOINT_SEQUENCE_NUMBER_KEY), + DynamoUtils.safeGetLong(dynamoRecord, CHECKPOINT_SUBSEQUENCE_NUMBER_KEY)) + ); + result.parentShardIds(DynamoUtils.safeGetSS(dynamoRecord, PARENT_SHARD_ID_KEY)); + + if (!Strings.isNullOrEmpty(DynamoUtils.safeGetString(dynamoRecord, PENDING_CHECKPOINT_SEQUENCE_KEY))) { + result.pendingCheckpoint( + new ExtendedSequenceNumber( + DynamoUtils.safeGetString(dynamoRecord, PENDING_CHECKPOINT_SEQUENCE_KEY), + DynamoUtils.safeGetLong(dynamoRecord, PENDING_CHECKPOINT_SUBSEQUENCE_KEY)) + ); + } + + return result; + } + + @Override + public Map getDynamoHashKey(final String leaseKey) { + Map result = new HashMap<>(); + + result.put(LEASE_KEY_KEY, DynamoUtils.createAttributeValue(leaseKey)); + + return result; + } + + @Override + public Map getDynamoHashKey(final Lease lease) { + return getDynamoHashKey(lease.leaseKey()); + } + + @Override + public Map getDynamoLeaseCounterExpectation(final Lease lease) { + return getDynamoLeaseCounterExpectation(lease.leaseCounter()); + } + + public Map getDynamoLeaseCounterExpectation(final Long leaseCounter) { + Map result = new HashMap<>(); + + ExpectedAttributeValue eav = new ExpectedAttributeValue(DynamoUtils.createAttributeValue(leaseCounter)); + result.put(LEASE_COUNTER_KEY, eav); + + return result; + } + + @Override + public Map getDynamoLeaseOwnerExpectation(final Lease lease) { + Map result = new HashMap<>(); + + ExpectedAttributeValue eav; + + if (lease.leaseOwner() == null) { + eav = new ExpectedAttributeValue(false); + } else { + eav = new ExpectedAttributeValue(DynamoUtils.createAttributeValue(lease.leaseOwner())); + } + + result.put(LEASE_OWNER_KEY, eav); + + return result; + } + + @Override + public Map getDynamoNonexistantExpectation() { + Map result = new HashMap<>(); + + ExpectedAttributeValue expectedAV = new ExpectedAttributeValue(false); + result.put(LEASE_KEY_KEY, expectedAV); + + return result; + } + + @Override + public Map getDynamoLeaseCounterUpdate(final Lease lease) { + return getDynamoLeaseCounterUpdate(lease.leaseCounter()); + } + + public Map getDynamoLeaseCounterUpdate(Long leaseCounter) { + Map result = new HashMap<>(); + + AttributeValueUpdate avu = + new AttributeValueUpdate(DynamoUtils.createAttributeValue(leaseCounter + 1), AttributeAction.PUT); + result.put(LEASE_COUNTER_KEY, avu); + + return result; + } + + @Override + public Map getDynamoTakeLeaseUpdate(final Lease lease, String owner) { + Map result = new HashMap<>(); + + result.put(LEASE_OWNER_KEY, new AttributeValueUpdate(DynamoUtils.createAttributeValue(owner), + AttributeAction.PUT)); + + String oldOwner = lease.leaseOwner(); + if (oldOwner != null && !oldOwner.equals(owner)) { + result.put(OWNER_SWITCHES_KEY, new AttributeValueUpdate(DynamoUtils.createAttributeValue(1L), + AttributeAction.ADD)); + } + + return result; + } + + @Override + public Map getDynamoEvictLeaseUpdate(final Lease lease) { + Map result = new HashMap<>(); + + result.put(LEASE_OWNER_KEY, new AttributeValueUpdate(null, AttributeAction.DELETE)); + + return result; + } + + @Override + public Map getDynamoUpdateLeaseUpdate(final Lease lease) { + Map result = new HashMap<>(); + result.put(CHECKPOINT_SEQUENCE_NUMBER_KEY, new AttributeValueUpdate(DynamoUtils.createAttributeValue(lease.checkpoint().getSequenceNumber()), + AttributeAction.PUT)); + result.put(CHECKPOINT_SUBSEQUENCE_NUMBER_KEY, new AttributeValueUpdate(DynamoUtils.createAttributeValue(lease.checkpoint().getSubSequenceNumber()), + AttributeAction.PUT)); + result.put(OWNER_SWITCHES_KEY, + new AttributeValueUpdate(DynamoUtils.createAttributeValue(lease.ownerSwitchesSinceCheckpoint()), + AttributeAction.PUT)); + + if (lease.pendingCheckpoint() != null && !lease.pendingCheckpoint().getSequenceNumber().isEmpty()) { + result.put(PENDING_CHECKPOINT_SEQUENCE_KEY, new AttributeValueUpdate(DynamoUtils.createAttributeValue(lease.pendingCheckpoint().getSequenceNumber()), AttributeAction.PUT)); + result.put(PENDING_CHECKPOINT_SUBSEQUENCE_KEY, new AttributeValueUpdate(DynamoUtils.createAttributeValue(lease.pendingCheckpoint().getSubSequenceNumber()), AttributeAction.PUT)); + } else { + result.put(PENDING_CHECKPOINT_SEQUENCE_KEY, new AttributeValueUpdate().withAction(AttributeAction.DELETE)); + result.put(PENDING_CHECKPOINT_SUBSEQUENCE_KEY, new AttributeValueUpdate().withAction(AttributeAction.DELETE)); + } + return result; + } + + @Override + public Collection getKeySchema() { + List keySchema = new ArrayList<>(); + keySchema.add(new KeySchemaElement().withAttributeName(LEASE_KEY_KEY).withKeyType(KeyType.HASH)); + + return keySchema; + } + + @Override + public Collection getAttributeDefinitions() { + List definitions = new ArrayList<>(); + definitions.add(new AttributeDefinition().withAttributeName(LEASE_KEY_KEY) + .withAttributeType(ScalarAttributeType.S)); + + return definitions; + } +} diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/DynamoDBLeaseTaker.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseTaker.java similarity index 85% rename from amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/DynamoDBLeaseTaker.java rename to amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseTaker.java index 138cc79a..e2ce5af4 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/DynamoDBLeaseTaker.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseTaker.java @@ -12,7 +12,7 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package software.amazon.kinesis.leases; +package software.amazon.kinesis.leases.dynamodb; import java.util.ArrayList; import java.util.Collection; @@ -27,20 +27,23 @@ import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; import com.amazonaws.services.cloudwatch.model.StandardUnit; + +import lombok.extern.slf4j.Slf4j; +import software.amazon.kinesis.leases.Lease; +import software.amazon.kinesis.leases.LeaseRefresher; +import software.amazon.kinesis.leases.LeaseTaker; import software.amazon.kinesis.leases.exceptions.DependencyException; import software.amazon.kinesis.leases.exceptions.InvalidStateException; import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException; -import software.amazon.kinesis.metrics.MetricsHelper; import software.amazon.kinesis.metrics.IMetricsScope; +import software.amazon.kinesis.metrics.MetricsHelper; import software.amazon.kinesis.metrics.MetricsLevel; -import lombok.extern.slf4j.Slf4j; - /** - * An implementation of ILeaseTaker that uses DynamoDB via LeaseManager. + * An implementation of {@link LeaseTaker} that uses DynamoDB via {@link LeaseRefresher}. */ @Slf4j -public class DynamoDBLeaseTaker implements LeaseTaker { +public class DynamoDBLeaseTaker implements LeaseTaker { private static final int TAKE_RETRIES = 3; private static final int SCAN_RETRIES = 1; @@ -53,17 +56,17 @@ public class DynamoDBLeaseTaker implements LeaseTaker { } }; - private final LeaseManager leaseManager; + private final LeaseRefresher leaseRefresher; private final String workerIdentifier; - private final Map allLeases = new HashMap(); + private final Map allLeases = new HashMap<>(); private final long leaseDurationNanos; private int maxLeasesForWorker = Integer.MAX_VALUE; private int maxLeasesToStealAtOneTime = 1; private long lastScanTimeNanos = 0L; - public DynamoDBLeaseTaker(LeaseManager leaseManager, String workerIdentifier, long leaseDurationMillis) { - this.leaseManager = leaseManager; + public DynamoDBLeaseTaker(LeaseRefresher leaseRefresher, String workerIdentifier, long leaseDurationMillis) { + this.leaseRefresher = leaseRefresher; this.workerIdentifier = workerIdentifier; this.leaseDurationNanos = TimeUnit.MILLISECONDS.toNanos(leaseDurationMillis); } @@ -81,7 +84,7 @@ public class DynamoDBLeaseTaker implements LeaseTaker { * @param maxLeasesForWorker Max leases this Worker can handle at a time * @return LeaseTaker */ - public DynamoDBLeaseTaker withMaxLeasesForWorker(int maxLeasesForWorker) { + public DynamoDBLeaseTaker withMaxLeasesForWorker(int maxLeasesForWorker) { if (maxLeasesForWorker <= 0) { throw new IllegalArgumentException("maxLeasesForWorker should be >= 1"); } @@ -97,7 +100,7 @@ public class DynamoDBLeaseTaker implements LeaseTaker { * @param maxLeasesToStealAtOneTime Steal up to this many leases at one time (for load balancing) * @return LeaseTaker */ - public DynamoDBLeaseTaker withMaxLeasesToStealAtOneTime(int maxLeasesToStealAtOneTime) { + public DynamoDBLeaseTaker withMaxLeasesToStealAtOneTime(int maxLeasesToStealAtOneTime) { if (maxLeasesToStealAtOneTime <= 0) { throw new IllegalArgumentException("maxLeasesToStealAtOneTime should be >= 1"); } @@ -109,7 +112,7 @@ public class DynamoDBLeaseTaker implements LeaseTaker { * {@inheritDoc} */ @Override - public Map takeLeases() throws DependencyException, InvalidStateException { + public Map takeLeases() throws DependencyException, InvalidStateException { return takeLeases(SYSTEM_CLOCK_CALLABLE); } @@ -125,10 +128,10 @@ public class DynamoDBLeaseTaker implements LeaseTaker { * @throws DependencyException * @throws InvalidStateException */ - synchronized Map takeLeases(Callable timeProvider) + synchronized Map takeLeases(Callable timeProvider) throws DependencyException, InvalidStateException { // Key is leaseKey - Map takenLeases = new HashMap(); + Map takenLeases = new HashMap<>(); long startTime = System.currentTimeMillis(); boolean success = false; @@ -159,21 +162,21 @@ public class DynamoDBLeaseTaker implements LeaseTaker { return takenLeases; } - List expiredLeases = getExpiredLeases(); + List expiredLeases = getExpiredLeases(); - Set leasesToTake = computeLeasesToTake(expiredLeases); - Set untakenLeaseKeys = new HashSet(); + Set leasesToTake = computeLeasesToTake(expiredLeases); + Set untakenLeaseKeys = new HashSet<>(); - for (T lease : leasesToTake) { - String leaseKey = lease.getLeaseKey(); + for (Lease lease : leasesToTake) { + String leaseKey = lease.leaseKey(); startTime = System.currentTimeMillis(); success = false; try { for (int i = 1; i <= TAKE_RETRIES; i++) { try { - if (leaseManager.takeLease(lease, workerIdentifier)) { - lease.setLastCounterIncrementNanos(System.nanoTime()); + if (leaseRefresher.takeLease(lease, workerIdentifier)) { + lease.lastCounterIncrementNanos(System.nanoTime()); takenLeases.put(leaseKey, lease); } else { untakenLeaseKeys.add(leaseKey); @@ -247,7 +250,7 @@ public class DynamoDBLeaseTaker implements LeaseTaker { */ private void updateAllLeases(Callable timeProvider) throws DependencyException, InvalidStateException, ProvisionedThroughputException { - List freshList = leaseManager.listLeases(); + List freshList = leaseRefresher.listLeases(); try { lastScanTimeNanos = timeProvider.call(); } catch (Exception e) { @@ -255,29 +258,29 @@ public class DynamoDBLeaseTaker implements LeaseTaker { } // This set will hold the lease keys not updated by the previous listLeases call. - Set notUpdated = new HashSet(allLeases.keySet()); + Set notUpdated = new HashSet<>(allLeases.keySet()); // Iterate over all leases, finding ones to try to acquire that haven't changed since the last iteration - for (T lease : freshList) { - String leaseKey = lease.getLeaseKey(); + for (Lease lease : freshList) { + String leaseKey = lease.leaseKey(); - T oldLease = allLeases.get(leaseKey); + Lease oldLease = allLeases.get(leaseKey); allLeases.put(leaseKey, lease); notUpdated.remove(leaseKey); if (oldLease != null) { // If we've seen this lease before... - if (oldLease.getLeaseCounter().equals(lease.getLeaseCounter())) { + if (oldLease.leaseCounter().equals(lease.leaseCounter())) { // ...and the counter hasn't changed, propagate the lastRenewalNanos time from the old lease - lease.setLastCounterIncrementNanos(oldLease.getLastCounterIncrementNanos()); + lease.lastCounterIncrementNanos(oldLease.lastCounterIncrementNanos()); } else { // ...and the counter has changed, set lastRenewalNanos to the time of the scan. - lease.setLastCounterIncrementNanos(lastScanTimeNanos); + lease.lastCounterIncrementNanos(lastScanTimeNanos); } } else { - if (lease.getLeaseOwner() == null) { + if (lease.leaseOwner() == null) { // if this new lease is unowned, it's never been renewed. - lease.setLastCounterIncrementNanos(0L); + lease.lastCounterIncrementNanos(0L); if (log.isDebugEnabled()) { log.debug("Treating new lease with key {} as never renewed because it is new and unowned.", @@ -285,7 +288,7 @@ public class DynamoDBLeaseTaker implements LeaseTaker { } } else { // if this new lease is owned, treat it as renewed as of the scan - lease.setLastCounterIncrementNanos(lastScanTimeNanos); + lease.lastCounterIncrementNanos(lastScanTimeNanos); if (log.isDebugEnabled()) { log.debug("Treating new lease with key {} as recently renewed because it is new and owned.", leaseKey); @@ -303,10 +306,10 @@ public class DynamoDBLeaseTaker implements LeaseTaker { /** * @return list of leases that were expired as of our last scan. */ - private List getExpiredLeases() { - List expiredLeases = new ArrayList(); + private List getExpiredLeases() { + List expiredLeases = new ArrayList<>(); - for (T lease : allLeases.values()) { + for (Lease lease : allLeases.values()) { if (lease.isExpired(leaseDurationNanos, lastScanTimeNanos)) { expiredLeases.add(lease); } @@ -318,13 +321,12 @@ public class DynamoDBLeaseTaker implements LeaseTaker { /** * Compute the number of leases I should try to take based on the state of the system. * - * @param allLeases map of shardId to lease containing all leases * @param expiredLeases list of leases we determined to be expired * @return set of leases to take. */ - private Set computeLeasesToTake(List expiredLeases) { + private Set computeLeasesToTake(List expiredLeases) { Map leaseCounts = computeLeaseCounts(expiredLeases); - Set leasesToTake = new HashSet(); + Set leasesToTake = new HashSet<>(); IMetricsScope metrics = MetricsHelper.getMetricsScope(); int numLeases = allLeases.size(); @@ -383,13 +385,13 @@ public class DynamoDBLeaseTaker implements LeaseTaker { } } else { // If there are no expired leases and we need a lease, consider stealing. - List leasesToSteal = chooseLeasesToSteal(leaseCounts, numLeasesToReachTarget, target); - for (T leaseToSteal : leasesToSteal) { + List leasesToSteal = chooseLeasesToSteal(leaseCounts, numLeasesToReachTarget, target); + for (Lease leaseToSteal : leasesToSteal) { log.info("Worker {} needed {} leases but none were expired, so it will steal lease {} from {}", workerIdentifier, numLeasesToReachTarget, - leaseToSteal.getLeaseKey(), - leaseToSteal.getLeaseOwner()); + leaseToSteal.leaseKey(), + leaseToSteal.leaseOwner()); leasesToTake.add(leaseToSteal); } } @@ -428,8 +430,8 @@ public class DynamoDBLeaseTaker implements LeaseTaker { * @param target target # of leases per worker * @return Leases to steal, or empty list if we should not steal */ - private List chooseLeasesToSteal(Map leaseCounts, int needed, int target) { - List leasesToSteal = new ArrayList<>(); + private List chooseLeasesToSteal(Map leaseCounts, int needed, int target) { + List leasesToSteal = new ArrayList<>(); Entry mostLoadedWorker = null; // Find the most loaded worker @@ -476,10 +478,10 @@ public class DynamoDBLeaseTaker implements LeaseTaker { } String mostLoadedWorkerIdentifier = mostLoadedWorker.getKey(); - List candidates = new ArrayList(); + List candidates = new ArrayList<>(); // Collect leases belonging to that worker - for (T lease : allLeases.values()) { - if (mostLoadedWorkerIdentifier.equals(lease.getLeaseOwner())) { + for (Lease lease : allLeases.values()) { + if (mostLoadedWorkerIdentifier.equals(lease.leaseOwner())) { candidates.add(lease); } } @@ -499,13 +501,13 @@ public class DynamoDBLeaseTaker implements LeaseTaker { * @param expiredLeases list of leases that are currently expired * @return map of workerIdentifier to lease count */ - private Map computeLeaseCounts(List expiredLeases) { - Map leaseCounts = new HashMap(); + private Map computeLeaseCounts(List expiredLeases) { + Map leaseCounts = new HashMap<>(); // Compute the number of leases per worker by looking through allLeases and ignoring leases that have expired. - for (T lease : allLeases.values()) { + for (Lease lease : allLeases.values()) { if (!expiredLeases.contains(lease)) { - String leaseOwner = lease.getLeaseOwner(); + String leaseOwner = lease.leaseOwner(); Integer oldCount = leaseCounts.get(leaseOwner); if (oldCount == null) { leaseCounts.put(leaseOwner, 1); diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/BlockOnParentShardTask.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/BlockOnParentShardTask.java index 81ef5588..1bbaaf05 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/BlockOnParentShardTask.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/BlockOnParentShardTask.java @@ -20,8 +20,8 @@ import lombok.AccessLevel; import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import software.amazon.kinesis.leases.LeaseManager; -import software.amazon.kinesis.leases.KinesisClientLease; +import software.amazon.kinesis.leases.Lease; +import software.amazon.kinesis.leases.LeaseRefresher; import software.amazon.kinesis.leases.ShardInfo; import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; @@ -39,7 +39,7 @@ import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; public class BlockOnParentShardTask implements ITask { @NonNull private final ShardInfo shardInfo; - private final LeaseManager leaseManager; + private final LeaseRefresher leaseRefresher; // Sleep for this duration if the parent shards have not completed processing, or we encounter an exception. private final long parentShardPollIntervalMillis; @@ -58,9 +58,9 @@ public class BlockOnParentShardTask implements ITask { try { boolean blockedOnParentShard = false; for (String shardId : shardInfo.parentShardIds()) { - KinesisClientLease lease = leaseManager.getLease(shardId); + Lease lease = leaseRefresher.getLease(shardId); if (lease != null) { - ExtendedSequenceNumber checkpoint = lease.getCheckpoint(); + ExtendedSequenceNumber checkpoint = lease.checkpoint(); if ((checkpoint == null) || (!checkpoint.equals(ExtendedSequenceNumber.SHARD_END))) { log.debug("Shard {} is not yet done. Its current checkpoint is {}", shardId, checkpoint); blockedOnParentShard = true; diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ConsumerStates.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ConsumerStates.java index ba8bb9d0..e66bc9d7 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ConsumerStates.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ConsumerStates.java @@ -191,7 +191,7 @@ class ConsumerStates { @Override public ITask createTask(ShardConsumer consumer) { return new BlockOnParentShardTask(consumer.shardInfo(), - consumer.leaseManager(), + consumer.leaseRefresher(), consumer.parentShardPollIntervalMillis()); } @@ -320,7 +320,7 @@ class ConsumerStates { consumer.recordProcessorCheckpointer(), consumer.taskBackoffTimeMillis(), consumer.skipShardSyncAtWorkerInitializationIfLeasesExist(), - consumer.leaseManagerProxy(), + consumer.shardDetector(), throttlingReporter, recordsFetcher.getRecords(), consumer.shouldCallProcessRecordsEvenForEmptyRecordList(), @@ -528,14 +528,14 @@ class ConsumerStates { public ITask createTask(ShardConsumer consumer) { // TODO: set shutdown reason return new ShutdownTask(consumer.shardInfo(), - consumer.leaseManagerProxy(), + consumer.shardDetector(), consumer.recordProcessor(), consumer.recordProcessorCheckpointer(), consumer.shutdownReason(), consumer.initialPositionInStream(), consumer.cleanupLeasesOfCompletedShards(), consumer.ignoreUnexpectedChildShards(), - consumer.leaseManager(), + consumer.leaseRefresher(), consumer.taskBackoffTimeMillis(), consumer.getRecordsCache()); } diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ProcessTask.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ProcessTask.java index 612c5c53..0972124f 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ProcessTask.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ProcessTask.java @@ -20,7 +20,7 @@ import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import software.amazon.kinesis.checkpoint.RecordProcessorCheckpointer; -import software.amazon.kinesis.leases.LeaseManagerProxy; +import software.amazon.kinesis.leases.ShardDetector; import software.amazon.kinesis.leases.ShardInfo; import software.amazon.kinesis.metrics.IMetricsScope; import software.amazon.kinesis.metrics.MetricsHelper; @@ -77,7 +77,7 @@ public class ProcessTask implements ITask { @NonNull final RecordProcessorCheckpointer recordProcessorCheckpointer, final long backoffTimeMillis, final boolean skipShardSyncAtWorkerInitializationIfLeasesExist, - final LeaseManagerProxy leaseManagerProxy, + final ShardDetector shardDetector, @NonNull final ThrottlingReporter throttlingReporter, final ProcessRecordsInput processRecordsInput, final boolean shouldCallProcessRecordsEvenForEmptyRecordList, @@ -93,7 +93,7 @@ public class ProcessTask implements ITask { Optional currentShard = Optional.empty(); if (!skipShardSyncAtWorkerInitializationIfLeasesExist) { - currentShard = leaseManagerProxy.listShards().stream() + currentShard = shardDetector.listShards().stream() .filter(shard -> shardInfo.shardId().equals(shard.getShardId())) .findFirst(); } diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShardConsumer.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShardConsumer.java index 1c2af227..46678d1b 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShardConsumer.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShardConsumer.java @@ -35,9 +35,8 @@ import lombok.Synchronized; import lombok.experimental.Accessors; import lombok.extern.slf4j.Slf4j; import software.amazon.kinesis.checkpoint.RecordProcessorCheckpointer; -import software.amazon.kinesis.leases.LeaseManager; -import software.amazon.kinesis.leases.KinesisClientLease; -import software.amazon.kinesis.leases.LeaseManagerProxy; +import software.amazon.kinesis.leases.LeaseRefresher; +import software.amazon.kinesis.leases.ShardDetector; import software.amazon.kinesis.leases.ShardInfo; import software.amazon.kinesis.metrics.IMetricsFactory; import software.amazon.kinesis.metrics.MetricsCollectingTaskDecorator; @@ -60,7 +59,7 @@ public class ShardConsumer { @NonNull private final String streamName; @NonNull - private final LeaseManager leaseManager; + private final LeaseRefresher leaseRefresher; @NonNull private final ExecutorService executorService; @NonNull @@ -87,7 +86,7 @@ public class ShardConsumer { private final boolean cleanupLeasesOfCompletedShards; private final boolean ignoreUnexpectedChildShards; @NonNull - private final LeaseManagerProxy leaseManagerProxy; + private final ShardDetector shardDetector; @NonNull private final IMetricsFactory metricsFactory; diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShardConsumerShutdownNotification.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShardConsumerShutdownNotification.java index fa423cca..7fe94141 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShardConsumerShutdownNotification.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShardConsumerShutdownNotification.java @@ -16,9 +16,9 @@ package software.amazon.kinesis.lifecycle; import java.util.concurrent.CountDownLatch; -import software.amazon.kinesis.processor.ShutdownNotificationAware; -import software.amazon.kinesis.leases.KinesisClientLease; +import software.amazon.kinesis.leases.Lease; import software.amazon.kinesis.leases.LeaseCoordinator; +import software.amazon.kinesis.processor.ShutdownNotificationAware; /** * Contains callbacks for completion of stages in a requested record processor shutdown. @@ -26,8 +26,8 @@ import software.amazon.kinesis.leases.LeaseCoordinator; */ public class ShardConsumerShutdownNotification implements ShutdownNotification { - private final LeaseCoordinator leaseCoordinator; - private final KinesisClientLease lease; + private final LeaseCoordinator leaseCoordinator; + private final Lease lease; private final CountDownLatch shutdownCompleteLatch; private final CountDownLatch notificationCompleteLatch; @@ -48,8 +48,10 @@ public class ShardConsumerShutdownNotification implements ShutdownNotification { * @param shutdownCompleteLatch * used to inform the caller once the record processor is fully shutdown */ - public ShardConsumerShutdownNotification(LeaseCoordinator leaseCoordinator, KinesisClientLease lease, - CountDownLatch notificationCompleteLatch, CountDownLatch shutdownCompleteLatch) { + public ShardConsumerShutdownNotification(final LeaseCoordinator leaseCoordinator, + final Lease lease, + final CountDownLatch notificationCompleteLatch, + final CountDownLatch shutdownCompleteLatch) { this.leaseCoordinator = leaseCoordinator; this.lease = lease; this.notificationCompleteLatch = notificationCompleteLatch; diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShutdownTask.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShutdownTask.java index 2d6b9430..d563f70e 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShutdownTask.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShutdownTask.java @@ -21,9 +21,8 @@ import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import software.amazon.kinesis.checkpoint.RecordProcessorCheckpointer; -import software.amazon.kinesis.leases.LeaseManager; -import software.amazon.kinesis.leases.KinesisClientLease; -import software.amazon.kinesis.leases.LeaseManagerProxy; +import software.amazon.kinesis.leases.LeaseRefresher; +import software.amazon.kinesis.leases.ShardDetector; import software.amazon.kinesis.leases.ShardInfo; import software.amazon.kinesis.leases.ShardSyncer; import software.amazon.kinesis.metrics.MetricsHelper; @@ -43,7 +42,7 @@ public class ShutdownTask implements ITask { @NonNull private final ShardInfo shardInfo; @NonNull - private final LeaseManagerProxy leaseManagerProxy; + private final ShardDetector shardDetector; @NonNull private final RecordProcessor recordProcessor; @NonNull @@ -55,7 +54,7 @@ public class ShutdownTask implements ITask { private final boolean cleanupLeasesOfCompletedShards; private final boolean ignoreUnexpectedChildShards; @NonNull - private final LeaseManager leaseManager; + private final LeaseRefresher leaseRefresher; private final long backoffTimeMillis; @NonNull private final GetRecordsCache getRecordsCache; @@ -114,8 +113,8 @@ public class ShutdownTask implements ITask { if (reason == ShutdownReason.TERMINATE) { log.debug("Looking for child shards of shard {}", shardInfo.shardId()); // create leases for the child shards - ShardSyncer.checkAndCreateLeasesForNewShards(leaseManagerProxy, - leaseManager, + ShardSyncer.checkAndCreateLeasesForNewShards(shardDetector, + leaseRefresher, initialPositionInStream, cleanupLeasesOfCompletedShards, ignoreUnexpectedChildShards); diff --git a/amazon-kinesis-client/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/CheckpointImplTestBase.java b/amazon-kinesis-client/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/CheckpointImplTestBase.java index 6283380a..b89e47de 100644 --- a/amazon-kinesis-client/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/CheckpointImplTestBase.java +++ b/amazon-kinesis-client/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/CheckpointImplTestBase.java @@ -95,7 +95,7 @@ public abstract class CheckpointImplTestBase { } /** - * Test method to verify setCheckpoint and getCheckpoint methods. + * Test method to verify checkpoint and checkpoint methods. * * @throws Exception */ diff --git a/amazon-kinesis-client/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/InMemoryCheckpointer.java b/amazon-kinesis-client/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/InMemoryCheckpointer.java index 09e512cc..39d290d9 100644 --- a/amazon-kinesis-client/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/InMemoryCheckpointer.java +++ b/amazon-kinesis-client/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/checkpoint/InMemoryCheckpointer.java @@ -108,7 +108,7 @@ public class InMemoryCheckpointer implements Checkpointer { @Override public ExtendedSequenceNumber getCheckpoint(String shardId) throws KinesisClientLibException { ExtendedSequenceNumber checkpoint = flushpoints.get(shardId); - log.debug("getCheckpoint shardId: {} checkpoint: {}", shardId, checkpoint); + log.debug("checkpoint shardId: {} checkpoint: {}", shardId, checkpoint); return checkpoint; } diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/coordinator/SchedulerTest.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/coordinator/SchedulerTest.java index 8bfd3ede..b7371ed2 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/coordinator/SchedulerTest.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/coordinator/SchedulerTest.java @@ -50,14 +50,12 @@ import com.amazonaws.services.kinesis.clientlibrary.exceptions.KinesisClientLibN import software.amazon.kinesis.checkpoint.Checkpoint; import software.amazon.kinesis.checkpoint.CheckpointConfig; import software.amazon.kinesis.checkpoint.CheckpointFactory; -import software.amazon.kinesis.leases.LeaseManager; -import software.amazon.kinesis.leases.KinesisClientLease; -import software.amazon.kinesis.leases.KinesisClientLibLeaseCoordinator; +import software.amazon.kinesis.leases.dynamodb.DynamoDBLeaseRefresher; import software.amazon.kinesis.leases.LeaseCoordinator; import software.amazon.kinesis.leases.LeaseManagementConfig; import software.amazon.kinesis.leases.LeaseManagementFactory; -import software.amazon.kinesis.leases.DynamoDBLeaseManager; -import software.amazon.kinesis.leases.LeaseManagerProxy; +import software.amazon.kinesis.leases.LeaseRefresher; +import software.amazon.kinesis.leases.ShardDetector; import software.amazon.kinesis.leases.ShardInfo; import software.amazon.kinesis.leases.ShardSyncTaskManager; import software.amazon.kinesis.lifecycle.InitializationInput; @@ -69,9 +67,9 @@ import software.amazon.kinesis.lifecycle.ShutdownReason; import software.amazon.kinesis.metrics.IMetricsFactory; import software.amazon.kinesis.metrics.MetricsConfig; import software.amazon.kinesis.processor.Checkpointer; -import software.amazon.kinesis.processor.RecordProcessor; import software.amazon.kinesis.processor.ProcessorConfig; import software.amazon.kinesis.processor.ProcessorFactory; +import software.amazon.kinesis.processor.RecordProcessor; import software.amazon.kinesis.retrieval.GetRecordsCache; import software.amazon.kinesis.retrieval.RetrievalConfig; import software.amazon.kinesis.retrieval.RetrievalFactory; @@ -107,13 +105,13 @@ public class SchedulerTest { @Mock private GetRecordsCache getRecordsCache; @Mock - private KinesisClientLibLeaseCoordinator leaseCoordinator; + private LeaseCoordinator leaseCoordinator; @Mock private ShardSyncTaskManager shardSyncTaskManager; @Mock - private DynamoDBLeaseManager dynamoDBLeaseManager; + private DynamoDBLeaseRefresher dynamoDBLeaseRefresher; @Mock - private LeaseManagerProxy leaseManagerProxy; + private ShardDetector shardDetector; @Mock private Checkpointer checkpoint; @@ -131,8 +129,8 @@ public class SchedulerTest { processorConfig = new ProcessorConfig(processorFactory); retrievalConfig = new RetrievalConfig(streamName, amazonKinesis).retrievalFactory(retrievalFactory); - when(leaseCoordinator.leaseManager()).thenReturn(dynamoDBLeaseManager); - when(shardSyncTaskManager.leaseManagerProxy()).thenReturn(leaseManagerProxy); + when(leaseCoordinator.leaseRefresher()).thenReturn(dynamoDBLeaseRefresher); + when(shardSyncTaskManager.shardDetector()).thenReturn(shardDetector); when(retrievalFactory.createGetRecordsCache(any(ShardInfo.class), any(IMetricsFactory.class))).thenReturn(getRecordsCache); scheduler = new Scheduler(checkpointConfig, coordinatorConfig, leaseManagementConfig, lifecycleConfig, @@ -236,11 +234,11 @@ public class SchedulerTest { @Test public final void testInitializationFailureWithRetries() throws Exception { doNothing().when(leaseCoordinator).initialize(); - when(leaseManagerProxy.listShards()).thenThrow(new RuntimeException()); + when(shardDetector.listShards()).thenThrow(new RuntimeException()); scheduler.run(); - verify(leaseManagerProxy, times(Scheduler.MAX_INITIALIZATION_ATTEMPTS)).listShards(); + verify(shardDetector, times(Scheduler.MAX_INITIALIZATION_ATTEMPTS)).listShards(); } @@ -250,9 +248,9 @@ public class SchedulerTest { final BigInteger startSeqNum = BigInteger.ONE; List shardList = KinesisLocalFileDataCreator.createShardList(numShards, kinesisShardPrefix, startSeqNum); Assert.assertEquals(numShards, shardList.size()); - List initialLeases = new ArrayList(); + List initialLeases = new ArrayList(); for (Shard shard : shardList) { - KinesisClientLease lease = ShardSyncer.newKCLLease(shard); + Lease lease = ShardSyncer.newKCLLease(shard); lease.setCheckpoint(ExtendedSequenceNumber.AT_TIMESTAMP); initialLeases.add(lease); } @@ -261,7 +259,7 @@ public class SchedulerTest { private void runAndTestWorker(List shardList, int threadPoolSize, - List initialLeases, + List initialLeases, int numberOfRecordsPerShard) throws Exception { File file = KinesisLocalFileDataCreator.generateTempDataFile(shardList, numberOfRecordsPerShard, "unitTestWT001"); IKinesisProxy fileBasedProxy = new KinesisLocalFileProxy(file.getAbsolutePath()); @@ -288,7 +286,7 @@ public class SchedulerTest { file.delete(); } - private SchedulerThread runWorker(final List initialLeases) throws Exception { + private SchedulerThread runWorker(final List initialLeases) throws Exception { final int maxRecords = 2; final long leaseDurationMillis = 10000L; @@ -296,16 +294,16 @@ public class SchedulerTest { final long idleTimeInMilliseconds = 2L; AmazonDynamoDB ddbClient = DynamoDBEmbedded.create().amazonDynamoDB(); - LeaseManager leaseManager = new KinesisClientLeaseManager("foo", ddbClient); - leaseManager.createLeaseTableIfNotExists(1L, 1L); - for (KinesisClientLease initialLease : initialLeases) { - leaseManager.createLeaseIfNotExists(initialLease); + LeaseManager leaseRefresher = new LeaseManager("foo", ddbClient); + leaseRefresher.createLeaseTableIfNotExists(1L, 1L); + for (Lease initialLease : initialLeases) { + leaseRefresher.createLeaseIfNotExists(initialLease); } checkpointConfig = new CheckpointConfig("foo", ddbClient, workerIdentifier) .failoverTimeMillis(leaseDurationMillis) .epsilonMillis(epsilonMillis) - .leaseManager(leaseManager); + .leaseRefresher(leaseRefresher); leaseManagementConfig = new LeaseManagementConfig("foo", ddbClient, amazonKinesis, streamName, workerIdentifier) .failoverTimeMillis(leaseDurationMillis) .epsilonMillis(epsilonMillis); @@ -323,7 +321,7 @@ public class SchedulerTest { private void testWorker(List shardList, int threadPoolSize, - List initialLeases, + List initialLeases, int numberOfRecordsPerShard, IKinesisProxy kinesisProxy, TestStreamletFactory recordProcessorFactory) throws Exception { @@ -442,25 +440,20 @@ public class SchedulerTest { } @Override - public DynamoDBLeaseManager createLeaseManager() { - return dynamoDBLeaseManager; + public DynamoDBLeaseRefresher createLeaseRefresher() { + return dynamoDBLeaseRefresher; } @Override - public KinesisClientLibLeaseCoordinator createKinesisClientLibLeaseCoordinator() { - return leaseCoordinator; - } - - @Override - public LeaseManagerProxy createLeaseManagerProxy() { - return leaseManagerProxy; + public ShardDetector createShardDetector() { + return shardDetector; } } private class TestKinesisCheckpointFactory implements CheckpointFactory { @Override - public Checkpointer createCheckpointer(final LeaseCoordinator leaseCoordinator, - final LeaseManager leaseManager) { + public Checkpointer createCheckpointer(final LeaseCoordinator leaseCoordinator, + final LeaseRefresher leaseRefresher) { return checkpoint; } } diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/coordinator/WorkerTest.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/coordinator/WorkerTest.java index eba7c600..97ccfbe5 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/coordinator/WorkerTest.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/coordinator/WorkerTest.java @@ -50,7 +50,7 @@ public class WorkerTest { @Mock private KinesisClientLibLeaseCoordinator leaseCoordinator; @Mock - private ILeaseManager leaseManager; + private ILeaseManager leaseRefresher; @Mock private software.amazon.kinesis.processor.IRecordProcessorFactory v1RecordProcessorFactory; @Mock @@ -151,7 +151,7 @@ public class WorkerTest { final String dummyKinesisShardId = "kinesis-0-0"; ExecutorService execService = null; - when(leaseCoordinator.leaseManager()).thenReturn(leaseManager); + when(leaseCoordinator.leaseRefresher()).thenReturn(leaseRefresher); Worker worker = new Worker(stageName, @@ -195,7 +195,7 @@ public class WorkerTest { ExecutorService execService = null; - when(leaseCoordinator.leaseManager()).thenReturn(leaseManager); + when(leaseCoordinator.leaseRefresher()).thenReturn(leaseRefresher); List initialState = createShardInfoList(ExtendedSequenceNumber.TRIM_HORIZON); List firstCheckpoint = createShardInfoList(new ExtendedSequenceNumber("1000")); @@ -270,7 +270,7 @@ public class WorkerTest { final String dummyKinesisShardId = "kinesis-0-0"; final String anotherDummyKinesisShardId = "kinesis-0-1"; ExecutorService execService = null; - when(leaseCoordinator.leaseManager()).thenReturn(leaseManager); + when(leaseCoordinator.leaseRefresher()).thenReturn(leaseRefresher); Worker worker = new Worker(stageName, @@ -325,7 +325,7 @@ public class WorkerTest { maxRecords, idleTimeInMilliseconds, callProcessRecordsForEmptyRecordList, skipCheckpointValidationValue, INITIAL_POSITION_LATEST); - when(leaseCoordinator.leaseManager()).thenReturn(leaseManager); + when(leaseCoordinator.leaseRefresher()).thenReturn(leaseRefresher); ExecutorService execService = Executors.newSingleThreadExecutor(); long shardPollInterval = 0L; Worker worker = @@ -392,7 +392,7 @@ public class WorkerTest { List shardList = createShardListWithOneSplit(); List initialLeases = new ArrayList(); KinesisClientLease lease = ShardSyncer.newKCLLease(shardList.get(0)); - lease.setCheckpoint(new ExtendedSequenceNumber("2")); + lease.checkpoint(new ExtendedSequenceNumber("2")); initialLeases.add(lease); runAndTestWorker(shardList, threadPoolSize, initialLeases, callProcessRecordsForEmptyRecordList, numberOfRecordsPerShard, config); } @@ -408,7 +408,7 @@ public class WorkerTest { List shardList = createShardListWithOneSplit(); List initialLeases = new ArrayList(); KinesisClientLease lease = ShardSyncer.newKCLLease(shardList.get(0)); - lease.setCheckpoint(new ExtendedSequenceNumber("2")); + lease.checkpoint(new ExtendedSequenceNumber("2")); initialLeases.add(lease); boolean callProcessRecordsForEmptyRecordList = true; RecordsFetcherFactory recordsFetcherFactory = new SimpleRecordsFetcherFactory(); @@ -500,7 +500,7 @@ public class WorkerTest { final List initialLeases = new ArrayList(); for (Shard shard : shardList) { KinesisClientLease lease = ShardSyncer.newKCLLease(shard); - lease.setCheckpoint(ExtendedSequenceNumber.TRIM_HORIZON); + lease.checkpoint(ExtendedSequenceNumber.TRIM_HORIZON); initialLeases.add(lease); } @@ -576,7 +576,7 @@ public class WorkerTest { final List initialLeases = new ArrayList(); for (Shard shard : shardList) { KinesisClientLease lease = ShardSyncer.newKCLLease(shard); - lease.setCheckpoint(ExtendedSequenceNumber.TRIM_HORIZON); + lease.checkpoint(ExtendedSequenceNumber.TRIM_HORIZON); initialLeases.add(lease); } @@ -674,16 +674,16 @@ public class WorkerTest { IMetricsFactory metricsFactory = mock(IMetricsFactory.class); ExtendedSequenceNumber checkpoint = new ExtendedSequenceNumber("123", 0L); - KinesisClientLeaseBuilder builder = new KinesisClientLeaseBuilder().withCheckpoint(checkpoint) - .withConcurrencyToken(UUID.randomUUID()).withLastCounterIncrementNanos(0L).withLeaseCounter(0L) - .withOwnerSwitchesSinceCheckpoint(0L).withLeaseOwner("Self"); + KinesisClientLeaseBuilder builder = new KinesisClientLeaseBuilder().checkpoint(checkpoint) + .concurrencyToken(UUID.randomUUID()).lastCounterIncrementNanos(0L).leaseCounter(0L) + .ownerSwitchesSinceCheckpoint(0L).leaseOwner("Self"); final List leases = new ArrayList<>(); final List currentAssignments = new ArrayList<>(); - KinesisClientLease lease = builder.withLeaseKey(String.format("shardId-%03d", 1)).build(); + KinesisClientLease lease = builder.leaseKey(String.format("shardId-%03d", 1)).build(); leases.add(lease); - currentAssignments.add(new ShardInfo(lease.getLeaseKey(), lease.getConcurrencyToken().toString(), - lease.getParentShardIds(), lease.getCheckpoint())); + currentAssignments.add(new ShardInfo(lease.leaseKey(), lease.concurrencyToken().toString(), + lease.parentShardIds(), lease.checkpoint())); when(leaseCoordinator.getAssignments()).thenAnswer(new Answer>() { @@ -762,16 +762,16 @@ public class WorkerTest { IMetricsFactory metricsFactory = mock(IMetricsFactory.class); ExtendedSequenceNumber checkpoint = new ExtendedSequenceNumber("123", 0L); - KinesisClientLeaseBuilder builder = new KinesisClientLeaseBuilder().withCheckpoint(checkpoint) - .withConcurrencyToken(UUID.randomUUID()).withLastCounterIncrementNanos(0L).withLeaseCounter(0L) - .withOwnerSwitchesSinceCheckpoint(0L).withLeaseOwner("Self"); + KinesisClientLeaseBuilder builder = new KinesisClientLeaseBuilder().checkpoint(checkpoint) + .concurrencyToken(UUID.randomUUID()).lastCounterIncrementNanos(0L).leaseCounter(0L) + .ownerSwitchesSinceCheckpoint(0L).leaseOwner("Self"); final List leases = new ArrayList<>(); final List currentAssignments = new ArrayList<>(); - KinesisClientLease lease = builder.withLeaseKey(String.format("shardId-%03d", 1)).build(); + KinesisClientLease lease = builder.leaseKey(String.format("shardId-%03d", 1)).build(); leases.add(lease); - currentAssignments.add(new ShardInfo(lease.getLeaseKey(), lease.getConcurrencyToken().toString(), - lease.getParentShardIds(), lease.getCheckpoint())); + currentAssignments.add(new ShardInfo(lease.leaseKey(), lease.concurrencyToken().toString(), + lease.parentShardIds(), lease.checkpoint())); when(leaseCoordinator.getAssignments()).thenAnswer(new Answer>() { @Override @@ -827,16 +827,16 @@ public class WorkerTest { IMetricsFactory metricsFactory = mock(IMetricsFactory.class); ExtendedSequenceNumber checkpoint = new ExtendedSequenceNumber("123", 0L); - KinesisClientLeaseBuilder builder = new KinesisClientLeaseBuilder().withCheckpoint(checkpoint) - .withConcurrencyToken(UUID.randomUUID()).withLastCounterIncrementNanos(0L).withLeaseCounter(0L) - .withOwnerSwitchesSinceCheckpoint(0L).withLeaseOwner("Self"); + KinesisClientLeaseBuilder builder = new KinesisClientLeaseBuilder().checkpoint(checkpoint) + .concurrencyToken(UUID.randomUUID()).lastCounterIncrementNanos(0L).leaseCounter(0L) + .ownerSwitchesSinceCheckpoint(0L).leaseOwner("Self"); final List leases = new ArrayList<>(); final List currentAssignments = new ArrayList<>(); - KinesisClientLease lease = builder.withLeaseKey(String.format("shardId-%03d", 1)).build(); + KinesisClientLease lease = builder.leaseKey(String.format("shardId-%03d", 1)).build(); leases.add(lease); - currentAssignments.add(new ShardInfo(lease.getLeaseKey(), lease.getConcurrencyToken().toString(), - lease.getParentShardIds(), lease.getCheckpoint())); + currentAssignments.add(new ShardInfo(lease.leaseKey(), lease.concurrencyToken().toString(), + lease.parentShardIds(), lease.checkpoint())); when(leaseCoordinator.getAssignments()).thenAnswer(new Answer>() { @Override @@ -1220,16 +1220,16 @@ public class WorkerTest { IMetricsFactory metricsFactory = mock(IMetricsFactory.class); ExtendedSequenceNumber checkpoint = new ExtendedSequenceNumber("123", 0L); - KinesisClientLeaseBuilder builder = new KinesisClientLeaseBuilder().withCheckpoint(checkpoint) - .withConcurrencyToken(UUID.randomUUID()).withLastCounterIncrementNanos(0L).withLeaseCounter(0L) - .withOwnerSwitchesSinceCheckpoint(0L).withLeaseOwner("Self"); + KinesisClientLeaseBuilder builder = new KinesisClientLeaseBuilder().checkpoint(checkpoint) + .concurrencyToken(UUID.randomUUID()).lastCounterIncrementNanos(0L).leaseCounter(0L) + .ownerSwitchesSinceCheckpoint(0L).leaseOwner("Self"); final List leases = new ArrayList<>(); final List currentAssignments = new ArrayList<>(); - KinesisClientLease lease = builder.withLeaseKey(String.format("shardId-%03d", 1)).build(); + KinesisClientLease lease = builder.leaseKey(String.format("shardId-%03d", 1)).build(); leases.add(lease); - currentAssignments.add(new ShardInfo(lease.getLeaseKey(), lease.getConcurrencyToken().toString(), - lease.getParentShardIds(), lease.getCheckpoint())); + currentAssignments.add(new ShardInfo(lease.leaseKey(), lease.concurrencyToken().toString(), + lease.parentShardIds(), lease.checkpoint())); when(leaseCoordinator.getAssignments()).thenAnswer(new Answer>() { @Override @@ -1304,16 +1304,16 @@ public class WorkerTest { IMetricsFactory metricsFactory = mock(IMetricsFactory.class); ExtendedSequenceNumber checkpoint = new ExtendedSequenceNumber("123", 0L); - KinesisClientLeaseBuilder builder = new KinesisClientLeaseBuilder().withCheckpoint(checkpoint) - .withConcurrencyToken(UUID.randomUUID()).withLastCounterIncrementNanos(0L).withLeaseCounter(0L) - .withOwnerSwitchesSinceCheckpoint(0L).withLeaseOwner("Self"); + KinesisClientLeaseBuilder builder = new KinesisClientLeaseBuilder().checkpoint(checkpoint) + .concurrencyToken(UUID.randomUUID()).lastCounterIncrementNanos(0L).leaseCounter(0L) + .ownerSwitchesSinceCheckpoint(0L).leaseOwner("Self"); final List leases = new ArrayList<>(); final List currentAssignments = new ArrayList<>(); - KinesisClientLease lease = builder.withLeaseKey(String.format("shardId-%03d", 1)).build(); + KinesisClientLease lease = builder.leaseKey(String.format("shardId-%03d", 1)).build(); leases.add(lease); - currentAssignments.add(new ShardInfo(lease.getLeaseKey(), lease.getConcurrencyToken().toString(), - lease.getParentShardIds(), lease.getCheckpoint())); + currentAssignments.add(new ShardInfo(lease.leaseKey(), lease.concurrencyToken().toString(), + lease.parentShardIds(), lease.checkpoint())); when(leaseCoordinator.getAssignments()).thenAnswer(new Answer>() { @Override @@ -1447,11 +1447,11 @@ public class WorkerTest { final IRecordProcessor processor = mock(IRecordProcessor.class); ExtendedSequenceNumber checkpoint = new ExtendedSequenceNumber("123", 0L); - KinesisClientLeaseBuilder builder = new KinesisClientLeaseBuilder().withCheckpoint(checkpoint) - .withConcurrencyToken(UUID.randomUUID()).withLastCounterIncrementNanos(0L).withLeaseCounter(0L) - .withOwnerSwitchesSinceCheckpoint(0L).withLeaseOwner("Self"); + KinesisClientLeaseBuilder builder = new KinesisClientLeaseBuilder().checkpoint(checkpoint) + .concurrencyToken(UUID.randomUUID()).lastCounterIncrementNanos(0L).leaseCounter(0L) + .ownerSwitchesSinceCheckpoint(0L).leaseOwner("Self"); final List leases = new ArrayList<>(); - KinesisClientLease lease = builder.withLeaseKey(String.format("shardId-%03d", 1)).build(); + KinesisClientLease lease = builder.leaseKey(String.format("shardId-%03d", 1)).build(); leases.add(lease); doAnswer(new Answer() { @@ -1460,7 +1460,7 @@ public class WorkerTest { workerInitialized.countDown(); return true; } - }).when(leaseManager).waitUntilLeaseTableExists(anyLong(), anyLong()); + }).when(leaseRefresher).waitUntilLeaseTableExists(anyLong(), anyLong()); doAnswer(new Answer() { @Override public IRecordProcessor answer(InvocationOnMock invocation) throws Throwable { @@ -1469,9 +1469,9 @@ public class WorkerTest { } }).when(recordProcessorFactory).createProcessor(); - when(config.getWorkerIdentifier()).thenReturn("Self"); - when(leaseManager.listLeases()).thenReturn(leases); - when(leaseManager.renewLease(leases.get(0))).thenReturn(true); + when(config.workerIdentifier()).thenReturn("Self"); + when(leaseRefresher.listLeases()).thenReturn(leases); + when(leaseRefresher.renewLease(leases.get(0))).thenReturn(true); when(executorService.submit(Matchers.> any())) .thenAnswer(new ShutdownHandlingAnswer(taskFuture)); when(taskFuture.isDone()).thenReturn(true); @@ -1481,7 +1481,7 @@ public class WorkerTest { Worker worker = new Worker.Builder() .recordProcessorFactory(recordProcessorFactory) .config(config) - .leaseManager(leaseManager) + .leaseRefresher(leaseRefresher) .kinesisProxy(kinesisProxy) .execService(executorService) .workerStateChangeListener(workerStateChangeListener) @@ -1513,7 +1513,7 @@ public class WorkerTest { .config(config) .build(); - Assert.assertNotNull(worker.getLeaseCoordinator().leaseManager()); + Assert.assertNotNull(worker.getLeaseCoordinator().leaseRefresher()); } @SuppressWarnings("unchecked") @@ -1521,14 +1521,14 @@ public class WorkerTest { public void testBuilderWhenLeaseManagerIsSet() { IRecordProcessorFactory recordProcessorFactory = mock(IRecordProcessorFactory.class); // Create an instance of ILeaseManager for injection and validation - ILeaseManager leaseManager = (ILeaseManager) mock(ILeaseManager.class); + ILeaseManager leaseRefresher = (ILeaseManager) mock(ILeaseManager.class); Worker worker = new Worker.Builder() .recordProcessorFactory(recordProcessorFactory) .config(config) - .leaseManager(leaseManager) + .leaseRefresher(leaseRefresher) .build(); - Assert.assertSame(leaseManager, worker.getLeaseCoordinator().leaseManager()); + Assert.assertSame(leaseRefresher, worker.getLeaseCoordinator().leaseRefresher()); } private abstract class InjectableWorker extends Worker { @@ -1563,14 +1563,14 @@ public class WorkerTest { } private KinesisClientLease makeLease(ExtendedSequenceNumber checkpoint, int shardId) { - return new KinesisClientLeaseBuilder().withCheckpoint(checkpoint).withConcurrencyToken(UUID.randomUUID()) - .withLastCounterIncrementNanos(0L).withLeaseCounter(0L).withOwnerSwitchesSinceCheckpoint(0L) - .withLeaseOwner("Self").withLeaseKey(String.format("shardId-%03d", shardId)).build(); + return new KinesisClientLeaseBuilder().checkpoint(checkpoint).concurrencyToken(UUID.randomUUID()) + .lastCounterIncrementNanos(0L).leaseCounter(0L).ownerSwitchesSinceCheckpoint(0L) + .leaseOwner("Self").leaseKey(String.format("shardId-%03d", shardId)).build(); } private ShardInfo makeShardInfo(KinesisClientLease lease) { - return new ShardInfo(lease.getLeaseKey(), lease.getConcurrencyToken().toString(), lease.getParentShardIds(), - lease.getCheckpoint()); + return new ShardInfo(lease.leaseKey(), lease.concurrencyToken().toString(), lease.parentShardIds(), + lease.checkpoint()); } private static class ShutdownReasonMatcher extends TypeSafeDiagnosingMatcher { @@ -1785,7 +1785,7 @@ public class WorkerTest { List initialLeases = new ArrayList(); for (Shard shard : shardList) { KinesisClientLease lease = ShardSyncer.newKCLLease(shard); - lease.setCheckpoint(ExtendedSequenceNumber.AT_TIMESTAMP); + lease.checkpoint(ExtendedSequenceNumber.AT_TIMESTAMP); initialLeases.add(lease); } runAndTestWorker(shardList, threadPoolSize, initialLeases, callProcessRecordsForEmptyRecordList, numberOfRecordsPerShard, config); @@ -1842,14 +1842,14 @@ public class WorkerTest { final long idleTimeInMilliseconds = 2L; AmazonDynamoDB ddbClient = DynamoDBEmbedded.create().amazonDynamoDB(); - LeaseManager leaseManager = new KinesisClientLeaseManager("foo", ddbClient); - leaseManager.createLeaseTableIfNotExists(1L, 1L); + LeaseManager leaseRefresher = new KinesisClientLeaseManager("foo", ddbClient); + leaseRefresher.createLeaseTableIfNotExists(1L, 1L); for (KinesisClientLease initialLease : initialLeases) { - leaseManager.createLeaseIfNotExists(initialLease); + leaseRefresher.createLeaseIfNotExists(initialLease); } KinesisClientLibLeaseCoordinator leaseCoordinator = - new KinesisClientLibLeaseCoordinator(leaseManager, + new KinesisClientLibLeaseCoordinator(leaseRefresher, stageName, leaseDurationMillis, epsilonMillis, diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/ExceptionThrowingLeaseManager.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/ExceptionThrowingLeaseRefresher.java similarity index 56% rename from amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/ExceptionThrowingLeaseManager.java rename to amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/ExceptionThrowingLeaseRefresher.java index b7c90bfa..08e4543f 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/ExceptionThrowingLeaseManager.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/ExceptionThrowingLeaseRefresher.java @@ -17,27 +17,27 @@ package software.amazon.kinesis.leases; import java.util.Arrays; import java.util.List; +import lombok.extern.slf4j.Slf4j; import software.amazon.kinesis.leases.exceptions.DependencyException; import software.amazon.kinesis.leases.exceptions.InvalidStateException; import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException; - -import lombok.extern.slf4j.Slf4j; +import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; /** - * Mock Lease Manager by randomly throwing Leasing Exceptions. + * Mock LeaseRefresher by randomly throwing Leasing Exceptions. * */ @Slf4j -public class ExceptionThrowingLeaseManager implements LeaseManager { +public class ExceptionThrowingLeaseRefresher implements LeaseRefresher { private static final Throwable EXCEPTION_MSG = new Throwable("Test Exception"); // Use array below to control in what situations we want to throw exceptions. - private int[] leaseManagerMethodCallingCount; + private int[] leaseRefresherMethodCallingCount; /** * Methods which we support (simulate exceptions). */ - public enum ExceptionThrowingLeaseManagerMethods { + public enum ExceptionThrowingLeaseRefresherMethods { CREATELEASETABLEIFNOTEXISTS(0), LEASETABLEEXISTS(1), WAITUNTILLEASETABLEEXISTS(2), @@ -54,7 +54,7 @@ public class ExceptionThrowingLeaseManager implements LeaseManager leaseManager; + // The real local lease refresher which would do the real implementations. + private final LeaseRefresher leaseRefresher; /** - * Constructor accepts lease manager as only argument. + * Constructor accepts lease refresher as only argument. * - * @param leaseManager which will do the real implementations + * @param leaseRefresher which will do the real implementations */ - ExceptionThrowingLeaseManager(LeaseManager leaseManager) { - this.leaseManager = leaseManager; - this.leaseManagerMethodCallingCount = new int[ExceptionThrowingLeaseManagerMethods.values().length]; + ExceptionThrowingLeaseRefresher(LeaseRefresher leaseRefresher) { + this.leaseRefresher = leaseRefresher; + this.leaseRefresherMethodCallingCount = new int[ExceptionThrowingLeaseRefresherMethods.values().length]; } /** @@ -86,7 +86,7 @@ public class ExceptionThrowingLeaseManager implements LeaseManager listLeases() + public List listLeases() throws DependencyException, InvalidStateException, ProvisionedThroughputException { - throwExceptions("listLeases", ExceptionThrowingLeaseManagerMethods.LISTLEASES); + throwExceptions("listLeases", ExceptionThrowingLeaseRefresherMethods.LISTLEASES); - return leaseManager.listLeases(); + return leaseRefresher.listLeases(); } @Override - public boolean createLeaseIfNotExists(KinesisClientLease lease) + public boolean createLeaseIfNotExists(Lease lease) throws DependencyException, InvalidStateException, ProvisionedThroughputException { - throwExceptions("createLeaseIfNotExists", ExceptionThrowingLeaseManagerMethods.CREATELEASEIFNOTEXISTS); + throwExceptions("createLeaseIfNotExists", ExceptionThrowingLeaseRefresherMethods.CREATELEASEIFNOTEXISTS); - return leaseManager.createLeaseIfNotExists(lease); + return leaseRefresher.createLeaseIfNotExists(lease); } @Override - public boolean renewLease(KinesisClientLease lease) + public boolean renewLease(Lease lease) throws DependencyException, InvalidStateException, ProvisionedThroughputException { - throwExceptions("renewLease", ExceptionThrowingLeaseManagerMethods.RENEWLEASE); + throwExceptions("renewLease", ExceptionThrowingLeaseRefresherMethods.RENEWLEASE); - return leaseManager.renewLease(lease); + return leaseRefresher.renewLease(lease); } @Override - public boolean takeLease(KinesisClientLease lease, String owner) + public boolean takeLease(Lease lease, String owner) throws DependencyException, InvalidStateException, ProvisionedThroughputException { - throwExceptions("takeLease", ExceptionThrowingLeaseManagerMethods.TAKELEASE); + throwExceptions("takeLease", ExceptionThrowingLeaseRefresherMethods.TAKELEASE); - return leaseManager.takeLease(lease, owner); + return leaseRefresher.takeLease(lease, owner); } @Override - public boolean evictLease(KinesisClientLease lease) + public boolean evictLease(Lease lease) throws DependencyException, InvalidStateException, ProvisionedThroughputException { - throwExceptions("evictLease", ExceptionThrowingLeaseManagerMethods.EVICTLEASE); + throwExceptions("evictLease", ExceptionThrowingLeaseRefresherMethods.EVICTLEASE); - return leaseManager.evictLease(lease); + return leaseRefresher.evictLease(lease); } @Override - public void deleteLease(KinesisClientLease lease) + public void deleteLease(Lease lease) throws DependencyException, InvalidStateException, ProvisionedThroughputException { - throwExceptions("deleteLease", ExceptionThrowingLeaseManagerMethods.DELETELEASE); + throwExceptions("deleteLease", ExceptionThrowingLeaseRefresherMethods.DELETELEASE); - leaseManager.deleteLease(lease); + leaseRefresher.deleteLease(lease); } @Override - public boolean updateLease(KinesisClientLease lease) + public boolean updateLease(Lease lease) throws DependencyException, InvalidStateException, ProvisionedThroughputException { - throwExceptions("updateLease", ExceptionThrowingLeaseManagerMethods.UPDATELEASE); + throwExceptions("updateLease", ExceptionThrowingLeaseRefresherMethods.UPDATELEASE); - return leaseManager.updateLease(lease); + return leaseRefresher.updateLease(lease); } @Override - public KinesisClientLease getLease(String shardId) + public Lease getLease(String shardId) throws DependencyException, InvalidStateException, ProvisionedThroughputException { - throwExceptions("getLease", ExceptionThrowingLeaseManagerMethods.GETLEASE); + throwExceptions("getLease", ExceptionThrowingLeaseRefresherMethods.GETLEASE); - return leaseManager.getLease(shardId); + return leaseRefresher.getLease(shardId); } @Override public void deleteAll() throws DependencyException, InvalidStateException, ProvisionedThroughputException { - throwExceptions("deleteAll", ExceptionThrowingLeaseManagerMethods.DELETEALL); + throwExceptions("deleteAll", ExceptionThrowingLeaseRefresherMethods.DELETEALL); - leaseManager.deleteAll(); + leaseRefresher.deleteAll(); } @Override @@ -215,4 +215,9 @@ public class ExceptionThrowingLeaseManager implements LeaseManager parentShardIds = new HashSet<>(); - - public KinesisClientLeaseBuilder withLeaseKey(String leaseKey) { - this.leaseKey = leaseKey; - return this; - } - - public KinesisClientLeaseBuilder withLeaseOwner(String leaseOwner) { - this.leaseOwner = leaseOwner; - return this; - } - - public KinesisClientLeaseBuilder withLeaseCounter(Long leaseCounter) { - this.leaseCounter = leaseCounter; - return this; - } - - public KinesisClientLeaseBuilder withConcurrencyToken(UUID concurrencyToken) { - this.concurrencyToken = concurrencyToken; - return this; - } - - public KinesisClientLeaseBuilder withLastCounterIncrementNanos(Long lastCounterIncrementNanos) { - this.lastCounterIncrementNanos = lastCounterIncrementNanos; - return this; - } - - public KinesisClientLeaseBuilder withCheckpoint(ExtendedSequenceNumber checkpoint) { - this.checkpoint = checkpoint; - return this; - } - - public KinesisClientLeaseBuilder withPendingCheckpoint(ExtendedSequenceNumber pendingCheckpoint) { - this.pendingCheckpoint = pendingCheckpoint; - return this; - } - - public KinesisClientLeaseBuilder withOwnerSwitchesSinceCheckpoint(Long ownerSwitchesSinceCheckpoint) { - this.ownerSwitchesSinceCheckpoint = ownerSwitchesSinceCheckpoint; - return this; - } - - public KinesisClientLeaseBuilder withParentShardIds(Set parentShardIds) { - this.parentShardIds = parentShardIds; - return this; - } - - public KinesisClientLease build() { - return new KinesisClientLease(leaseKey, leaseOwner, leaseCounter, concurrencyToken, lastCounterIncrementNanos, - checkpoint, pendingCheckpoint, ownerSwitchesSinceCheckpoint, parentShardIds); - } -} \ No newline at end of file diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/LeaseBuilder.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/LeaseBuilder.java new file mode 100644 index 00000000..3265a1ad --- /dev/null +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/LeaseBuilder.java @@ -0,0 +1,42 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package software.amazon.kinesis.leases; + +import java.util.HashSet; +import java.util.Set; +import java.util.UUID; + +import lombok.Setter; +import lombok.experimental.Accessors; +import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; + +@Setter +@Accessors(fluent = true) +public class LeaseBuilder { + private String leaseKey; + private String leaseOwner; + private Long leaseCounter = 0L; + private UUID concurrencyToken; + private Long lastCounterIncrementNanos; + private ExtendedSequenceNumber checkpoint; + private ExtendedSequenceNumber pendingCheckpoint; + private Long ownerSwitchesSinceCheckpoint = 0L; + private Set parentShardIds = new HashSet<>(); + + public Lease build() { + return new Lease(leaseKey, leaseOwner, leaseCounter, concurrencyToken, lastCounterIncrementNanos, + checkpoint, pendingCheckpoint, ownerSwitchesSinceCheckpoint, parentShardIds); + } +} \ No newline at end of file diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/LeaseCoordinatorExerciser.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/LeaseCoordinatorExerciser.java index 0a32a5d2..76d5dcd2 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/LeaseCoordinatorExerciser.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/LeaseCoordinatorExerciser.java @@ -30,17 +30,23 @@ import javax.swing.*; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; -import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; + +import lombok.extern.slf4j.Slf4j; +import software.amazon.kinesis.leases.dynamodb.DynamoDBLeaseCoordinator; +import software.amazon.kinesis.leases.dynamodb.DynamoDBLeaseRefresher; +import software.amazon.kinesis.leases.dynamodb.DynamoDBLeaseSerializer; import software.amazon.kinesis.leases.exceptions.DependencyException; import software.amazon.kinesis.leases.exceptions.InvalidStateException; import software.amazon.kinesis.leases.exceptions.LeasingException; import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException; import software.amazon.kinesis.metrics.CWMetricsFactory; - -import lombok.extern.slf4j.Slf4j; +import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; @Slf4j public class LeaseCoordinatorExerciser { + private static final int MAX_LEASES_FOR_WORKER = Integer.MAX_VALUE; + private static final int MAX_LEASES_TO_STEAL_AT_ONE_TIME = 1; + private static final int MAX_LEASE_RENEWER_THREAD_COUNT = 20; public static void main(String[] args) throws InterruptedException, DependencyException, InvalidStateException, ProvisionedThroughputException, @@ -55,38 +61,40 @@ public class LeaseCoordinatorExerciser { new DefaultAWSCredentialsProviderChain(); AmazonDynamoDBClient ddb = new AmazonDynamoDBClient(creds); - LeaseManager leaseManager = new KinesisClientDynamoDBLeaseManager("nagl_ShardProgress", ddb); + LeaseRefresher leaseRefresher = new DynamoDBLeaseRefresher("nagl_ShardProgress", ddb, new DynamoDBLeaseSerializer(), true); - if (leaseManager.createLeaseTableIfNotExists(10L, 50L)) { + if (leaseRefresher.createLeaseTableIfNotExists(10L, 50L)) { log.info("Waiting for newly created lease table"); - if (!leaseManager.waitUntilLeaseTableExists(10, 300)) { + if (!leaseRefresher.waitUntilLeaseTableExists(10, 300)) { log.error("Table was not created in time"); return; } } CWMetricsFactory metricsFactory = new CWMetricsFactory(creds, "testNamespace", 30 * 1000, 1000); - final List> coordinators = - new ArrayList>(); + final List coordinators = new ArrayList<>(); for (int i = 0; i < numCoordinators; i++) { String workerIdentifier = "worker-" + Integer.toString(i); - LeaseCoordinator coord = new LeaseCoordinator(leaseManager, + LeaseCoordinator coord = new DynamoDBLeaseCoordinator(leaseRefresher, workerIdentifier, leaseDurationMillis, epsilonMillis, + MAX_LEASES_FOR_WORKER, + MAX_LEASES_TO_STEAL_AT_ONE_TIME, + MAX_LEASE_RENEWER_THREAD_COUNT, metricsFactory); coordinators.add(coord); } - leaseManager.deleteAll(); + leaseRefresher.deleteAll(); for (int i = 0; i < numLeases; i++) { - KinesisClientLease lease = new KinesisClientLease(); - lease.setLeaseKey(Integer.toString(i)); - lease.setCheckpoint(new ExtendedSequenceNumber("checkpoint")); - leaseManager.createLeaseIfNotExists(lease); + Lease lease = new Lease(); + lease.leaseKey(Integer.toString(i)); + lease.checkpoint(new ExtendedSequenceNumber("checkpoint")); + leaseRefresher.createLeaseIfNotExists(lease); } final JFrame frame = new JFrame("Test Visualizer"); @@ -97,10 +105,10 @@ public class LeaseCoordinatorExerciser { frame.getContentPane().add(panel); final Map labels = new HashMap(); - for (final LeaseCoordinator coord : coordinators) { + for (final LeaseCoordinator coord : coordinators) { JPanel coordPanel = new JPanel(); coordPanel.setLayout(new BoxLayout(coordPanel, BoxLayout.X_AXIS)); - final Button button = new Button("Stop " + coord.getWorkerIdentifier()); + final Button button = new Button("Stop " + coord.workerIdentifier()); button.setMaximumSize(new Dimension(200, 50)); button.addActionListener(new ActionListener() { @@ -108,14 +116,14 @@ public class LeaseCoordinatorExerciser { public void actionPerformed(ActionEvent arg0) { if (coord.isRunning()) { coord.stop(); - button.setLabel("Start " + coord.getWorkerIdentifier()); + button.setLabel("Start " + coord.workerIdentifier()); } else { try { coord.start(); } catch (LeasingException e) { log.error("{}", e); } - button.setLabel("Stop " + coord.getWorkerIdentifier()); + button.setLabel("Stop " + coord.workerIdentifier()); } } @@ -124,7 +132,7 @@ public class LeaseCoordinatorExerciser { JLabel label = new JLabel(); coordPanel.add(label); - labels.put(coord.getWorkerIdentifier(), label); + labels.put(coord.workerIdentifier(), label); panel.add(coordPanel); } @@ -141,17 +149,17 @@ public class LeaseCoordinatorExerciser { @Override public void run() { while (true) { - for (LeaseCoordinator coord : coordinators) { - String workerIdentifier = coord.getWorkerIdentifier(); + for (LeaseCoordinator coord : coordinators) { + String workerIdentifier = coord.workerIdentifier(); JLabel label = labels.get(workerIdentifier); - List asgn = new ArrayList(coord.getAssignments()); - Collections.sort(asgn, new Comparator() { + List asgn = new ArrayList<>(coord.getAssignments()); + Collections.sort(asgn, new Comparator() { @Override - public int compare(KinesisClientLease arg0, KinesisClientLease arg1) { - return arg0.getLeaseKey().compareTo(arg1.getLeaseKey()); + public int compare(final Lease arg0, final Lease arg1) { + return arg0.leaseKey().compareTo(arg1.leaseKey()); } }); @@ -160,19 +168,19 @@ public class LeaseCoordinatorExerciser { builder.append(""); builder.append(workerIdentifier).append(":").append(asgn.size()).append(" "); - for (KinesisClientLease lease : asgn) { - String leaseKey = lease.getLeaseKey(); + for (Lease lease : asgn) { + String leaseKey = lease.leaseKey(); String lastOwner = lastOwners.get(leaseKey); // Color things green when they switch owners, decay the green-ness over time. Integer greenNess = greenNesses.get(leaseKey); - if (greenNess == null || lastOwner == null || !lastOwner.equals(lease.getLeaseOwner())) { + if (greenNess == null || lastOwner == null || !lastOwner.equals(lease.leaseOwner())) { greenNess = 200; } else { greenNess = Math.max(0, greenNess - 20); } greenNesses.put(leaseKey, greenNess); - lastOwners.put(leaseKey, lease.getLeaseOwner()); + lastOwners.put(leaseKey, lease.leaseOwner()); builder.append(String.format("%03d", String.format("#00%02x00", greenNess), @@ -203,7 +211,7 @@ public class LeaseCoordinatorExerciser { frame.pack(); frame.setVisible(true); - for (LeaseCoordinator coord : coordinators) { + for (LeaseCoordinator coord : coordinators) { coord.start(); } } diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/LeaseIntegrationTest.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/LeaseIntegrationTest.java index d71061df..acd6c1bc 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/LeaseIntegrationTest.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/LeaseIntegrationTest.java @@ -21,16 +21,19 @@ import org.junit.runner.Description; import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; + +import lombok.extern.slf4j.Slf4j; +import software.amazon.kinesis.leases.dynamodb.DynamoDBLeaseRefresher; +import software.amazon.kinesis.leases.dynamodb.DynamoDBLeaseSerializer; import software.amazon.kinesis.metrics.MetricsHelper; import software.amazon.kinesis.metrics.NullMetricsFactory; -import lombok.extern.slf4j.Slf4j; - -@Ignore @Slf4j +@Ignore public class LeaseIntegrationTest { + private LeaseSerializer leaseSerializer = new DynamoDBLeaseSerializer(); - protected static KinesisClientDynamoDBLeaseManager leaseManager; + protected static DynamoDBLeaseRefresher leaseRefresher; protected static AmazonDynamoDBClient ddbClient = new AmazonDynamoDBClient(new DefaultAWSCredentialsProviderChain()); @@ -39,25 +42,25 @@ public class LeaseIntegrationTest { @Override protected void starting(Description description) { - if (leaseManager == null) { + if (leaseRefresher == null) { // Do some static setup once per class. - leaseManager = new KinesisClientDynamoDBLeaseManager("nagl_ShardProgress", ddbClient, true); + leaseRefresher = new DynamoDBLeaseRefresher("nagl_ShardProgress", ddbClient, leaseSerializer, true); MetricsHelper.startScope(new NullMetricsFactory()); } try { - if (!leaseManager.leaseTableExists()) { + if (!leaseRefresher.leaseTableExists()) { log.info("Creating lease table"); - leaseManager.createLeaseTableIfNotExists(10L, 10L); + leaseRefresher.createLeaseTableIfNotExists(10L, 10L); - leaseManager.waitUntilLeaseTableExists(10, 500); + leaseRefresher.waitUntilLeaseTableExists(10, 500); } log.info("Beginning test case {}", description.getMethodName()); - for (KinesisClientLease lease : leaseManager.listLeases()) { - leaseManager.deleteLease(lease); + for (Lease lease : leaseRefresher.listLeases()) { + leaseRefresher.deleteLease(lease); } } catch (Exception e) { String message = @@ -69,3 +72,4 @@ public class LeaseIntegrationTest { }; } + diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/ShardSyncTaskIntegrationTest.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/ShardSyncTaskIntegrationTest.java index 973255ec..9fd9ff0d 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/ShardSyncTaskIntegrationTest.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/ShardSyncTaskIntegrationTest.java @@ -37,6 +37,8 @@ import com.amazonaws.services.kinesis.model.DescribeStreamSummaryRequest; import com.amazonaws.services.kinesis.model.Shard; import com.amazonaws.services.kinesis.model.StreamStatus; +import software.amazon.kinesis.leases.dynamodb.DynamoDBLeaseRefresher; +import software.amazon.kinesis.leases.dynamodb.DynamoDBLeaseSerializer; import software.amazon.kinesis.leases.exceptions.DependencyException; import software.amazon.kinesis.leases.exceptions.InvalidStateException; import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException; @@ -48,8 +50,8 @@ public class ShardSyncTaskIntegrationTest { private static final String STREAM_NAME = "IntegrationTestStream02"; private static AmazonKinesis amazonKinesis; - private KinesisClientLeaseManager leaseManager; - private LeaseManagerProxy leaseManagerProxy; + private LeaseRefresher leaseRefresher; + private ShardDetector shardDetector; /** * @throws java.lang.Exception @@ -85,12 +87,13 @@ public class ShardSyncTaskIntegrationTest { @Before public void setUp() throws Exception { boolean useConsistentReads = true; - leaseManager = - new KinesisClientDynamoDBLeaseManager("ShardSyncTaskIntegrationTest", + leaseRefresher = + new DynamoDBLeaseRefresher("ShardSyncTaskIntegrationTest", AmazonDynamoDBClientBuilder.standard().withRegion(Regions.US_EAST_1).build(), + new DynamoDBLeaseSerializer(), useConsistentReads); - leaseManagerProxy = new KinesisLeaseManagerProxy(amazonKinesis, STREAM_NAME, 500L, 50); + shardDetector = new KinesisShardDetector(amazonKinesis, STREAM_NAME, 500L, 50); } /** @@ -109,24 +112,24 @@ public class ShardSyncTaskIntegrationTest { */ @Test public final void testCall() throws DependencyException, InvalidStateException, ProvisionedThroughputException { - if (!leaseManager.leaseTableExists()) { + if (!leaseRefresher.leaseTableExists()) { final Long readCapacity = 10L; final Long writeCapacity = 10L; - leaseManager.createLeaseTableIfNotExists(readCapacity, writeCapacity); + leaseRefresher.createLeaseTableIfNotExists(readCapacity, writeCapacity); } - leaseManager.deleteAll(); - Set shardIds = leaseManagerProxy.listShards().stream().map(Shard::getShardId).collect(Collectors.toSet()); - ShardSyncTask syncTask = new ShardSyncTask(leaseManagerProxy, - leaseManager, + leaseRefresher.deleteAll(); + Set shardIds = shardDetector.listShards().stream().map(Shard::getShardId).collect(Collectors.toSet()); + ShardSyncTask syncTask = new ShardSyncTask(shardDetector, + leaseRefresher, InitialPositionInStreamExtended.newInitialPosition(InitialPositionInStream.LATEST), false, false, 0L); syncTask.call(); - List leases = leaseManager.listLeases(); - Set leaseKeys = new HashSet(); - for (KinesisClientLease lease : leases) { - leaseKeys.add(lease.getLeaseKey()); + List leases = leaseRefresher.listLeases(); + Set leaseKeys = new HashSet<>(); + for (Lease lease : leases) { + leaseKeys.add(lease.leaseKey()); } // Verify that all shardIds had leases for them diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/ShardSyncerTest.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/ShardSyncerTest.java index 106a4634..77b9f8f7 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/ShardSyncerTest.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/ShardSyncerTest.java @@ -50,7 +50,9 @@ import com.amazonaws.services.kinesis.model.SequenceNumberRange; import com.amazonaws.services.kinesis.model.Shard; import lombok.extern.slf4j.Slf4j; -import software.amazon.kinesis.leases.ExceptionThrowingLeaseManager.ExceptionThrowingLeaseManagerMethods; +import software.amazon.kinesis.leases.ExceptionThrowingLeaseRefresher.ExceptionThrowingLeaseRefresherMethods; +import software.amazon.kinesis.leases.dynamodb.DynamoDBLeaseRefresher; +import software.amazon.kinesis.leases.dynamodb.DynamoDBLeaseSerializer; import software.amazon.kinesis.leases.exceptions.DependencyException; import software.amazon.kinesis.leases.exceptions.InvalidStateException; import software.amazon.kinesis.leases.exceptions.LeasingException; @@ -70,30 +72,32 @@ public class ShardSyncerTest { InitialPositionInStreamExtended.newInitialPosition(InitialPositionInStream.TRIM_HORIZON); private static final InitialPositionInStreamExtended INITIAL_POSITION_AT_TIMESTAMP = InitialPositionInStreamExtended.newInitialPositionAtTimestamp(new Date(1000L)); + private static final int EXPONENT = 128; + private final boolean cleanupLeasesOfCompletedShards = true; private AmazonDynamoDB ddbClient = DynamoDBEmbedded.create().amazonDynamoDB(); - private DynamoDBLeaseManager dynamoDBLeaseManager = new KinesisClientDynamoDBLeaseManager("tempTestTable", ddbClient); - private static final int EXPONENT = 128; + private DynamoDBLeaseRefresher dynamoDBLeaseRefresher = new DynamoDBLeaseRefresher("tempTestTable", ddbClient, + new DynamoDBLeaseSerializer(), true); /** * Old/Obsolete max value of a sequence number (2^128 -1). */ public static final BigInteger MAX_SEQUENCE_NUMBER = new BigInteger("2").pow(EXPONENT).subtract(BigInteger.ONE); @Mock - private LeaseManagerProxy leaseManagerProxy; + private ShardDetector shardDetector; @Before public void setUp() throws Exception { - boolean created = dynamoDBLeaseManager.createLeaseTableIfNotExists(1L, 1L); + boolean created = dynamoDBLeaseRefresher.createLeaseTableIfNotExists(1L, 1L); if (created) { log.info("New table created."); } - dynamoDBLeaseManager.deleteAll(); + dynamoDBLeaseRefresher.deleteAll(); } @After public void tearDown() throws Exception { - dynamoDBLeaseManager.deleteAll(); + dynamoDBLeaseRefresher.deleteAll(); } /** @@ -102,7 +106,7 @@ public class ShardSyncerTest { @Test public final void testDetermineNewLeasesToCreateNoShards() { List shards = new ArrayList<>(); - List leases = new ArrayList<>(); + List leases = new ArrayList<>(); assertTrue(ShardSyncer.determineNewLeasesToCreate(shards, leases, INITIAL_POSITION_LATEST).isEmpty()); } @@ -113,7 +117,7 @@ public class ShardSyncerTest { @Test public final void testDetermineNewLeasesToCreate0Leases0Reshards() { List shards = new ArrayList<>(); - List currentLeases = new ArrayList<>(); + List currentLeases = new ArrayList<>(); SequenceNumberRange sequenceRange = ShardObjectHelper.newSequenceNumberRange("342980", null); String shardId0 = "shardId-0"; @@ -122,14 +126,14 @@ public class ShardSyncerTest { String shardId1 = "shardId-1"; shards.add(ShardObjectHelper.newShard(shardId1, null, null, sequenceRange)); - List newLeases = + List newLeases = ShardSyncer.determineNewLeasesToCreate(shards, currentLeases, INITIAL_POSITION_LATEST); assertEquals(2, newLeases.size()); Set expectedLeaseShardIds = new HashSet<>(); expectedLeaseShardIds.add(shardId0); expectedLeaseShardIds.add(shardId1); - for (KinesisClientLease lease : newLeases) { - assertTrue(expectedLeaseShardIds.contains(lease.getLeaseKey())); + for (Lease lease : newLeases) { + assertTrue(expectedLeaseShardIds.contains(lease.leaseKey())); } } @@ -140,7 +144,7 @@ public class ShardSyncerTest { @Test public final void testDetermineNewLeasesToCreate0Leases0Reshards1Inconsistent() { List shards = new ArrayList<>(); - List currentLeases = new ArrayList<>(); + List currentLeases = new ArrayList<>(); SequenceNumberRange sequenceRange = ShardObjectHelper.newSequenceNumberRange("342980", null); String shardId0 = "shardId-0"; @@ -155,14 +159,14 @@ public class ShardSyncerTest { Set inconsistentShardIds = new HashSet<>(); inconsistentShardIds.add(shardId2); - List newLeases = + List newLeases = ShardSyncer.determineNewLeasesToCreate(shards, currentLeases, INITIAL_POSITION_LATEST, inconsistentShardIds); assertEquals(2, newLeases.size()); Set expectedLeaseShardIds = new HashSet<>(); expectedLeaseShardIds.add(shardId0); expectedLeaseShardIds.add(shardId1); - for (KinesisClientLease lease : newLeases) { - assertTrue(expectedLeaseShardIds.contains(lease.getLeaseKey())); + for (Lease lease : newLeases) { + assertTrue(expectedLeaseShardIds.contains(lease.leaseKey())); } } @@ -211,20 +215,20 @@ public class ShardSyncerTest { IOException { List shards = constructShardListForGraphA(); - when(leaseManagerProxy.listShards()).thenReturn(shards); + when(shardDetector.listShards()).thenReturn(shards); - ShardSyncer.checkAndCreateLeasesForNewShards(leaseManagerProxy, dynamoDBLeaseManager, INITIAL_POSITION_LATEST, + ShardSyncer.checkAndCreateLeasesForNewShards(shardDetector, dynamoDBLeaseRefresher, INITIAL_POSITION_LATEST, cleanupLeasesOfCompletedShards); - List newLeases = dynamoDBLeaseManager.listLeases(); + List newLeases = dynamoDBLeaseRefresher.listLeases(); Set expectedLeaseShardIds = new HashSet<>(); expectedLeaseShardIds.add("shardId-4"); expectedLeaseShardIds.add("shardId-8"); expectedLeaseShardIds.add("shardId-9"); expectedLeaseShardIds.add("shardId-10"); assertEquals(expectedLeaseShardIds.size(), newLeases.size()); - for (KinesisClientLease lease1 : newLeases) { - assertTrue(expectedLeaseShardIds.contains(lease1.getLeaseKey())); - assertEquals(ExtendedSequenceNumber.LATEST, lease1.getCheckpoint()); + for (Lease lease1 : newLeases) { + assertTrue(expectedLeaseShardIds.contains(lease1.leaseKey())); + assertEquals(ExtendedSequenceNumber.LATEST, lease1.checkpoint()); } } @@ -243,19 +247,19 @@ public class ShardSyncerTest { IOException { List shards = constructShardListForGraphA(); - when(leaseManagerProxy.listShards()).thenReturn(shards); + when(shardDetector.listShards()).thenReturn(shards); - ShardSyncer.checkAndCreateLeasesForNewShards(leaseManagerProxy, dynamoDBLeaseManager, INITIAL_POSITION_TRIM_HORIZON, + ShardSyncer.checkAndCreateLeasesForNewShards(shardDetector, dynamoDBLeaseRefresher, INITIAL_POSITION_TRIM_HORIZON, cleanupLeasesOfCompletedShards); - List newLeases = dynamoDBLeaseManager.listLeases(); + List newLeases = dynamoDBLeaseRefresher.listLeases(); Set expectedLeaseShardIds = new HashSet<>(); for (int i = 0; i < 11; i++) { expectedLeaseShardIds.add("shardId-" + i); } assertEquals(expectedLeaseShardIds.size(), newLeases.size()); - for (KinesisClientLease lease1 : newLeases) { - assertTrue(expectedLeaseShardIds.contains(lease1.getLeaseKey())); - assertEquals(ExtendedSequenceNumber.TRIM_HORIZON, lease1.getCheckpoint()); + for (Lease lease1 : newLeases) { + assertTrue(expectedLeaseShardIds.contains(lease1.leaseKey())); + assertEquals(ExtendedSequenceNumber.TRIM_HORIZON, lease1.checkpoint()); } } @@ -272,19 +276,19 @@ public class ShardSyncerTest { ProvisionedThroughputException, IOException { List shards = constructShardListForGraphA(); - when(leaseManagerProxy.listShards()).thenReturn(shards); + when(shardDetector.listShards()).thenReturn(shards); - ShardSyncer.checkAndCreateLeasesForNewShards(leaseManagerProxy, dynamoDBLeaseManager, INITIAL_POSITION_AT_TIMESTAMP, + ShardSyncer.checkAndCreateLeasesForNewShards(shardDetector, dynamoDBLeaseRefresher, INITIAL_POSITION_AT_TIMESTAMP, cleanupLeasesOfCompletedShards); - List newLeases = dynamoDBLeaseManager.listLeases(); + List newLeases = dynamoDBLeaseRefresher.listLeases(); Set expectedLeaseShardIds = new HashSet<>(); for (int i = 0; i < 11; i++) { expectedLeaseShardIds.add("shardId-" + i); } assertEquals(expectedLeaseShardIds.size(), newLeases.size()); - for (KinesisClientLease lease1 : newLeases) { - assertTrue(expectedLeaseShardIds.contains(lease1.getLeaseKey())); - assertEquals(ExtendedSequenceNumber.AT_TIMESTAMP, lease1.getCheckpoint()); + for (Lease lease1 : newLeases) { + assertTrue(expectedLeaseShardIds.contains(lease1.leaseKey())); + assertEquals(ExtendedSequenceNumber.AT_TIMESTAMP, lease1.checkpoint()); } } @@ -304,9 +308,9 @@ public class ShardSyncerTest { range.setEndingSequenceNumber(null); shards.get(3).setSequenceNumberRange(range); - when(leaseManagerProxy.listShards()).thenReturn(shards); + when(shardDetector.listShards()).thenReturn(shards); - ShardSyncer.checkAndCreateLeasesForNewShards(leaseManagerProxy, dynamoDBLeaseManager, INITIAL_POSITION_TRIM_HORIZON, + ShardSyncer.checkAndCreateLeasesForNewShards(shardDetector, dynamoDBLeaseRefresher, INITIAL_POSITION_TRIM_HORIZON, cleanupLeasesOfCompletedShards); } @@ -328,19 +332,19 @@ public class ShardSyncerTest { range.setEndingSequenceNumber(null); shard.setSequenceNumberRange(range); - when(leaseManagerProxy.listShards()).thenReturn(shards); + when(shardDetector.listShards()).thenReturn(shards); - ShardSyncer.checkAndCreateLeasesForNewShards(leaseManagerProxy, dynamoDBLeaseManager, INITIAL_POSITION_LATEST, + ShardSyncer.checkAndCreateLeasesForNewShards(shardDetector, dynamoDBLeaseRefresher, INITIAL_POSITION_LATEST, cleanupLeasesOfCompletedShards, true); - List newLeases = dynamoDBLeaseManager.listLeases(); + List newLeases = dynamoDBLeaseRefresher.listLeases(); Set expectedLeaseShardIds = new HashSet<>(); expectedLeaseShardIds.add("shardId-4"); expectedLeaseShardIds.add("shardId-5"); expectedLeaseShardIds.add("shardId-8"); assertEquals(expectedLeaseShardIds.size(), newLeases.size()); - for (KinesisClientLease lease1 : newLeases) { - assertTrue(expectedLeaseShardIds.contains(lease1.getLeaseKey())); - assertEquals(ExtendedSequenceNumber.LATEST, lease1.getCheckpoint()); + for (Lease lease1 : newLeases) { + assertTrue(expectedLeaseShardIds.contains(lease1.leaseKey())); + assertEquals(ExtendedSequenceNumber.LATEST, lease1.checkpoint()); } } @@ -372,14 +376,14 @@ public class ShardSyncerTest { public final void testCheckAndCreateLeasesForNewShardsAtTrimHorizonAndClosedShardWithDeleteLeaseExceptions() throws KinesisClientLibIOException, DependencyException, InvalidStateException, ProvisionedThroughputException, IOException { - // Define the max calling count for lease manager methods. + // Define the max calling count for lease refresher methods. // From the Shard Graph, the max count of calling could be 10 int maxCallingCount = 10; for (int c = 1; c <= maxCallingCount; c = c + 2) { testCheckAndCreateLeasesForNewShardsAtSpecifiedPositionAndClosedShardImpl( - ExceptionThrowingLeaseManagerMethods.DELETELEASE, c, INITIAL_POSITION_TRIM_HORIZON); - // Need to clean up lease manager every time after calling ShardSyncer - dynamoDBLeaseManager.deleteAll(); + ExceptionThrowingLeaseRefresherMethods.DELETELEASE, c, INITIAL_POSITION_TRIM_HORIZON); + // Need to clean up lease refresher every time after calling ShardSyncer + dynamoDBLeaseRefresher.deleteAll(); } } @@ -396,14 +400,14 @@ public class ShardSyncerTest { public final void testCheckAndCreateLeasesForNewShardsAtTrimHorizonAndClosedShardWithListLeasesExceptions() throws KinesisClientLibIOException, DependencyException, InvalidStateException, ProvisionedThroughputException, IOException { - // Define the max calling count for lease manager methods. + // Define the max calling count for lease refresher methods. // From the Shard Graph, the max count of calling could be 10 int maxCallingCount = 10; for (int c = 1; c <= maxCallingCount; c = c + 2) { testCheckAndCreateLeasesForNewShardsAtSpecifiedPositionAndClosedShardImpl( - ExceptionThrowingLeaseManagerMethods.LISTLEASES, c, INITIAL_POSITION_TRIM_HORIZON); - // Need to clean up lease manager every time after calling ShardSyncer - dynamoDBLeaseManager.deleteAll(); + ExceptionThrowingLeaseRefresherMethods.LISTLEASES, c, INITIAL_POSITION_TRIM_HORIZON); + // Need to clean up lease refresher every time after calling ShardSyncer + dynamoDBLeaseRefresher.deleteAll(); } } @@ -420,34 +424,34 @@ public class ShardSyncerTest { public final void testCheckAndCreateLeasesForNewShardsAtTrimHorizonAndClosedShardWithCreateLeaseExceptions() throws KinesisClientLibIOException, DependencyException, InvalidStateException, ProvisionedThroughputException, IOException { - // Define the max calling count for lease manager methods. + // Define the max calling count for lease refresher methods. // From the Shard Graph, the max count of calling could be 10 int maxCallingCount = 5; for (int c = 1; c <= maxCallingCount; c = c + 2) { testCheckAndCreateLeasesForNewShardsAtSpecifiedPositionAndClosedShardImpl( - ExceptionThrowingLeaseManagerMethods.CREATELEASEIFNOTEXISTS, c,INITIAL_POSITION_TRIM_HORIZON); - // Need to clean up lease manager every time after calling ShardSyncer - dynamoDBLeaseManager.deleteAll(); + ExceptionThrowingLeaseRefresherMethods.CREATELEASEIFNOTEXISTS, c,INITIAL_POSITION_TRIM_HORIZON); + // Need to clean up lease refresher every time after calling ShardSyncer + dynamoDBLeaseRefresher.deleteAll(); } } - // Try catch leaseException for different lease manager methods and eventually let it succeed. + // Try catch leaseException for different lease refresher methods and eventually let it succeed. // This would not throw any exceptions if: // 1). exceptionMethod equals to null or NONE. // 2). exceptionTime is a very big or negative value. - private void retryCheckAndCreateLeaseForNewShards(ExceptionThrowingLeaseManagerMethods exceptionMethod, + private void retryCheckAndCreateLeaseForNewShards(ExceptionThrowingLeaseRefresherMethods exceptionMethod, int exceptionTime, InitialPositionInStreamExtended position) throws KinesisClientLibIOException, DependencyException, InvalidStateException, ProvisionedThroughputException { if (exceptionMethod != null) { - ExceptionThrowingLeaseManager exceptionThrowingLeaseManager = - new ExceptionThrowingLeaseManager(dynamoDBLeaseManager); - // Set exception and throwing time for exceptionThrowingManager. - exceptionThrowingLeaseManager.setLeaseLeaseManagerThrowingExceptionScenario(exceptionMethod, exceptionTime); + ExceptionThrowingLeaseRefresher exceptionThrowingLeaseRefresher = + new ExceptionThrowingLeaseRefresher(dynamoDBLeaseRefresher); + // Set exception and throwing time for exceptionThrowingRefresher. + exceptionThrowingLeaseRefresher.leaseRefresherThrowingExceptionScenario(exceptionMethod, exceptionTime); // Only need to try two times. for (int i = 1; i <= 2; i++) { try { - ShardSyncer.checkAndCreateLeasesForNewShards(leaseManagerProxy, - exceptionThrowingLeaseManager, + ShardSyncer.checkAndCreateLeasesForNewShards(shardDetector, + exceptionThrowingLeaseRefresher, position, cleanupLeasesOfCompletedShards); return; @@ -455,11 +459,11 @@ public class ShardSyncerTest { log.debug("Catch leasing exception", e); } // Clear throwing exception scenario every time after calling ShardSyncer - exceptionThrowingLeaseManager.clearLeaseManagerThrowingExceptionScenario(); + exceptionThrowingLeaseRefresher.clearLeaseRefresherThrowingExceptionScenario(); } } else { - ShardSyncer.checkAndCreateLeasesForNewShards(leaseManagerProxy, - dynamoDBLeaseManager, + ShardSyncer.checkAndCreateLeasesForNewShards(shardDetector, + dynamoDBLeaseRefresher, position, cleanupLeasesOfCompletedShards); } @@ -493,15 +497,15 @@ public class ShardSyncerTest { public final void testCheckAndCreateLeasesForNewShardsAtTimestampAndClosedShardWithDeleteLeaseExceptions() throws KinesisClientLibIOException, DependencyException, InvalidStateException, ProvisionedThroughputException, IOException { - // Define the max calling count for lease manager methods. + // Define the max calling count for lease refresher methods. // From the Shard Graph, the max count of calling could be 10 int maxCallingCount = 10; for (int c = 1; c <= maxCallingCount; c = c + 2) { testCheckAndCreateLeasesForNewShardsAtSpecifiedPositionAndClosedShardImpl( - ExceptionThrowingLeaseManagerMethods.DELETELEASE, + ExceptionThrowingLeaseRefresherMethods.DELETELEASE, c, INITIAL_POSITION_AT_TIMESTAMP); - // Need to clean up lease manager every time after calling ShardSyncer - dynamoDBLeaseManager.deleteAll(); + // Need to clean up lease refresher every time after calling ShardSyncer + dynamoDBLeaseRefresher.deleteAll(); } } @@ -516,15 +520,15 @@ public class ShardSyncerTest { public final void testCheckAndCreateLeasesForNewShardsAtTimestampAndClosedShardWithListLeasesExceptions() throws KinesisClientLibIOException, DependencyException, InvalidStateException, ProvisionedThroughputException, IOException { - // Define the max calling count for lease manager methods. + // Define the max calling count for lease refresher methods. // From the Shard Graph, the max count of calling could be 10 int maxCallingCount = 10; for (int c = 1; c <= maxCallingCount; c = c + 2) { testCheckAndCreateLeasesForNewShardsAtSpecifiedPositionAndClosedShardImpl( - ExceptionThrowingLeaseManagerMethods.LISTLEASES, + ExceptionThrowingLeaseRefresherMethods.LISTLEASES, c, INITIAL_POSITION_AT_TIMESTAMP); - // Need to clean up lease manager every time after calling ShardSyncer - dynamoDBLeaseManager.deleteAll(); + // Need to clean up lease refresher every time after calling ShardSyncer + dynamoDBLeaseRefresher.deleteAll(); } } @@ -539,21 +543,21 @@ public class ShardSyncerTest { public final void testCheckAndCreateLeasesForNewShardsAtTimestampAndClosedShardWithCreateLeaseExceptions() throws KinesisClientLibIOException, DependencyException, InvalidStateException, ProvisionedThroughputException, IOException { - // Define the max calling count for lease manager methods. + // Define the max calling count for lease refresher methods. // From the Shard Graph, the max count of calling could be 10 int maxCallingCount = 5; for (int c = 1; c <= maxCallingCount; c = c + 2) { testCheckAndCreateLeasesForNewShardsAtSpecifiedPositionAndClosedShardImpl( - ExceptionThrowingLeaseManagerMethods.CREATELEASEIFNOTEXISTS, + ExceptionThrowingLeaseRefresherMethods.CREATELEASEIFNOTEXISTS, c, INITIAL_POSITION_AT_TIMESTAMP); - // Need to clean up lease manager every time after calling ShardSyncer - dynamoDBLeaseManager.deleteAll(); + // Need to clean up lease refresher every time after calling ShardSyncer + dynamoDBLeaseRefresher.deleteAll(); } } - // Real implementation of testing CheckAndCreateLeasesForNewShards with different leaseManager types. + // Real implementation of testing CheckAndCreateLeasesForNewShards with different leaseRefresher types. private void testCheckAndCreateLeasesForNewShardsAtSpecifiedPositionAndClosedShardImpl( - ExceptionThrowingLeaseManagerMethods exceptionMethod, + ExceptionThrowingLeaseRefresherMethods exceptionMethod, int exceptionTime, InitialPositionInStreamExtended position) throws KinesisClientLibIOException, DependencyException, InvalidStateException, ProvisionedThroughputException, @@ -562,39 +566,39 @@ public class ShardSyncerTest { new ExtendedSequenceNumber(position.getInitialPositionInStream().toString()); List shards = constructShardListForGraphA(); - when(leaseManagerProxy.listShards()).thenReturn(shards); + when(shardDetector.listShards()).thenReturn(shards); retryCheckAndCreateLeaseForNewShards(exceptionMethod, exceptionTime, position); - List newLeases = dynamoDBLeaseManager.listLeases(); + List newLeases = dynamoDBLeaseRefresher.listLeases(); Map expectedShardIdToCheckpointMap = new HashMap<>(); for (int i = 0; i < 11; i++) { expectedShardIdToCheckpointMap.put("shardId-" + i, extendedSequenceNumber); } assertEquals(expectedShardIdToCheckpointMap.size(), newLeases.size()); - for (KinesisClientLease lease1 : newLeases) { - ExtendedSequenceNumber expectedCheckpoint = expectedShardIdToCheckpointMap.get(lease1.getLeaseKey()); + for (Lease lease1 : newLeases) { + ExtendedSequenceNumber expectedCheckpoint = expectedShardIdToCheckpointMap.get(lease1.leaseKey()); assertNotNull(expectedCheckpoint); - assertEquals(expectedCheckpoint, lease1.getCheckpoint()); + assertEquals(expectedCheckpoint, lease1.checkpoint()); } - KinesisClientLease closedShardLease = dynamoDBLeaseManager.getLease("shardId-0"); - closedShardLease.setCheckpoint(ExtendedSequenceNumber.SHARD_END); - dynamoDBLeaseManager.updateLease(closedShardLease); - expectedShardIdToCheckpointMap.remove(closedShardLease.getLeaseKey()); - KinesisClientLease childShardLease = dynamoDBLeaseManager.getLease("shardId-6"); - childShardLease.setCheckpoint(new ExtendedSequenceNumber("34290")); - dynamoDBLeaseManager.updateLease(childShardLease); - expectedShardIdToCheckpointMap.put(childShardLease.getLeaseKey(), new ExtendedSequenceNumber("34290")); + Lease closedShardLease = dynamoDBLeaseRefresher.getLease("shardId-0"); + closedShardLease.checkpoint(ExtendedSequenceNumber.SHARD_END); + dynamoDBLeaseRefresher.updateLease(closedShardLease); + expectedShardIdToCheckpointMap.remove(closedShardLease.leaseKey()); + Lease childShardLease = dynamoDBLeaseRefresher.getLease("shardId-6"); + childShardLease.checkpoint(new ExtendedSequenceNumber("34290")); + dynamoDBLeaseRefresher.updateLease(childShardLease); + expectedShardIdToCheckpointMap.put(childShardLease.leaseKey(), new ExtendedSequenceNumber("34290")); retryCheckAndCreateLeaseForNewShards(exceptionMethod, exceptionTime, position); - newLeases = dynamoDBLeaseManager.listLeases(); + newLeases = dynamoDBLeaseRefresher.listLeases(); assertEquals(expectedShardIdToCheckpointMap.size(), newLeases.size()); - for (KinesisClientLease lease1 : newLeases) { - ExtendedSequenceNumber expectedCheckpoint = expectedShardIdToCheckpointMap.get(lease1.getLeaseKey()); + for (Lease lease1 : newLeases) { + ExtendedSequenceNumber expectedCheckpoint = expectedShardIdToCheckpointMap.get(lease1.leaseKey()); assertNotNull(expectedCheckpoint); - assertEquals(expectedCheckpoint, lease1.getCheckpoint()); + assertEquals(expectedCheckpoint, lease1.checkpoint()); } } @@ -612,15 +616,15 @@ public class ShardSyncerTest { throws DependencyException, InvalidStateException, ProvisionedThroughputException, IOException, KinesisClientLibIOException { String garbageShardId = "shardId-garbage-001"; - KinesisClientLease garbageLease = ShardSyncer.newKCLLease(ShardObjectHelper.newShard(garbageShardId, + Lease garbageLease = ShardSyncer.newKCLLease(ShardObjectHelper.newShard(garbageShardId, null, null, ShardObjectHelper.newSequenceNumberRange("101", null))); - garbageLease.setCheckpoint(new ExtendedSequenceNumber("999")); - dynamoDBLeaseManager.createLeaseIfNotExists(garbageLease); - assertEquals(garbageShardId, dynamoDBLeaseManager.getLease(garbageShardId).getLeaseKey()); + garbageLease.checkpoint(new ExtendedSequenceNumber("999")); + dynamoDBLeaseRefresher.createLeaseIfNotExists(garbageLease); + assertEquals(garbageShardId, dynamoDBLeaseRefresher.getLease(garbageShardId).leaseKey()); testBootstrapShardLeasesAtStartingPosition(INITIAL_POSITION_LATEST); - assertNull(dynamoDBLeaseManager.getLease(garbageShardId)); + assertNull(dynamoDBLeaseRefresher.getLease(garbageShardId)); } private void testBootstrapShardLeasesAtStartingPosition(InitialPositionInStreamExtended initialPosition) @@ -634,19 +638,19 @@ public class ShardSyncerTest { String shardId1 = "shardId-1"; shards.add(ShardObjectHelper.newShard(shardId1, null, null, sequenceRange)); - when(leaseManagerProxy.listShards()).thenReturn(shards); + when(shardDetector.listShards()).thenReturn(shards); - ShardSyncer.bootstrapShardLeases(leaseManagerProxy, dynamoDBLeaseManager, initialPosition, cleanupLeasesOfCompletedShards, + ShardSyncer.bootstrapShardLeases(shardDetector, dynamoDBLeaseRefresher, initialPosition, cleanupLeasesOfCompletedShards, false); - List newLeases = dynamoDBLeaseManager.listLeases(); + List newLeases = dynamoDBLeaseRefresher.listLeases(); assertEquals(2, newLeases.size()); Set expectedLeaseShardIds = new HashSet(); expectedLeaseShardIds.add(shardId0); expectedLeaseShardIds.add(shardId1); - for (KinesisClientLease lease1 : newLeases) { - assertTrue(expectedLeaseShardIds.contains(lease1.getLeaseKey())); + for (Lease lease1 : newLeases) { + assertTrue(expectedLeaseShardIds.contains(lease1.leaseKey())); assertEquals(new ExtendedSequenceNumber(initialPosition.getInitialPositionInStream().toString()), - lease1.getCheckpoint()); + lease1.checkpoint()); } } @@ -656,7 +660,7 @@ public class ShardSyncerTest { @Test public final void testDetermineNewLeasesToCreateStartingPosition() { List shards = new ArrayList<>(); - List currentLeases = new ArrayList<>(); + List currentLeases = new ArrayList<>(); SequenceNumberRange sequenceRange = ShardObjectHelper.newSequenceNumberRange("342980", null); String shardId0 = "shardId-0"; @@ -670,16 +674,16 @@ public class ShardSyncerTest { initialPositions.add(INITIAL_POSITION_TRIM_HORIZON); for (InitialPositionInStreamExtended initialPosition : initialPositions) { - List newLeases = + List newLeases = ShardSyncer.determineNewLeasesToCreate(shards, currentLeases, initialPosition); assertEquals(2, newLeases.size()); Set expectedLeaseShardIds = new HashSet(); expectedLeaseShardIds.add(shardId0); expectedLeaseShardIds.add(shardId1); - for (KinesisClientLease lease : newLeases) { - assertTrue(expectedLeaseShardIds.contains(lease.getLeaseKey())); + for (Lease lease : newLeases) { + assertTrue(expectedLeaseShardIds.contains(lease.leaseKey())); assertEquals(new ExtendedSequenceNumber(initialPosition.getInitialPositionInStream().toString()), - lease.getCheckpoint()); + lease.checkpoint()); } } } @@ -690,7 +694,7 @@ public class ShardSyncerTest { @Test public final void testDetermineNewLeasesToCreateIgnoreClosedShard() { List shards = new ArrayList<>(); - List currentLeases = new ArrayList<>(); + List currentLeases = new ArrayList<>(); shards.add(ShardObjectHelper.newShard("shardId-0", null, @@ -702,10 +706,10 @@ public class ShardSyncerTest { null, ShardObjectHelper.newSequenceNumberRange("405", null))); - List newLeases = + List newLeases = ShardSyncer.determineNewLeasesToCreate(shards, currentLeases, INITIAL_POSITION_LATEST); assertEquals(1, newLeases.size()); - assertEquals(lastShardId, newLeases.get(0).getLeaseKey()); + assertEquals(lastShardId, newLeases.get(0).leaseKey()); } /** @@ -721,13 +725,13 @@ public class ShardSyncerTest { @Test public final void testDetermineNewLeasesToCreateSplitMergeLatest1() { List shards = constructShardListForGraphA(); - List currentLeases = new ArrayList<>(); + List currentLeases = new ArrayList<>(); currentLeases.add(newLease("shardId-3")); currentLeases.add(newLease("shardId-4")); currentLeases.add(newLease("shardId-5")); - List newLeases = + List newLeases = ShardSyncer.determineNewLeasesToCreate(shards, currentLeases, INITIAL_POSITION_LATEST); Map expectedShardIdCheckpointMap = new HashMap<>(); expectedShardIdCheckpointMap.put("shardId-8", ExtendedSequenceNumber.TRIM_HORIZON); @@ -738,10 +742,10 @@ public class ShardSyncerTest { expectedShardIdCheckpointMap.put("shardId-7", ExtendedSequenceNumber.TRIM_HORIZON); assertEquals(expectedShardIdCheckpointMap.size(), newLeases.size()); - for (KinesisClientLease lease : newLeases) { + for (Lease lease : newLeases) { assertTrue("Unexpected lease: " + lease, - expectedShardIdCheckpointMap.containsKey(lease.getLeaseKey())); - assertEquals(expectedShardIdCheckpointMap.get(lease.getLeaseKey()), lease.getCheckpoint()); + expectedShardIdCheckpointMap.containsKey(lease.leaseKey())); + assertEquals(expectedShardIdCheckpointMap.get(lease.leaseKey()), lease.checkpoint()); } } @@ -758,13 +762,13 @@ public class ShardSyncerTest { @Test public final void testDetermineNewLeasesToCreateSplitMergeLatest2() { List shards = constructShardListForGraphA(); - List currentLeases = new ArrayList<>(); + List currentLeases = new ArrayList<>(); currentLeases.add(newLease("shardId-4")); currentLeases.add(newLease("shardId-5")); currentLeases.add(newLease("shardId-7")); - List newLeases = + List newLeases = ShardSyncer.determineNewLeasesToCreate(shards, currentLeases, INITIAL_POSITION_LATEST); Map expectedShardIdCheckpointMap = new HashMap<>(); expectedShardIdCheckpointMap.put("shardId-8", ExtendedSequenceNumber.TRIM_HORIZON); @@ -773,10 +777,10 @@ public class ShardSyncerTest { expectedShardIdCheckpointMap.put("shardId-6", ExtendedSequenceNumber.LATEST); assertEquals(expectedShardIdCheckpointMap.size(), newLeases.size()); - for (KinesisClientLease lease : newLeases) { + for (Lease lease : newLeases) { assertTrue("Unexpected lease: " + lease, - expectedShardIdCheckpointMap.containsKey(lease.getLeaseKey())); - assertEquals(expectedShardIdCheckpointMap.get(lease.getLeaseKey()), lease.getCheckpoint()); + expectedShardIdCheckpointMap.containsKey(lease.leaseKey())); + assertEquals(expectedShardIdCheckpointMap.get(lease.leaseKey()), lease.checkpoint()); } } @@ -793,13 +797,13 @@ public class ShardSyncerTest { @Test public final void testDetermineNewLeasesToCreateSplitMergeHorizon1() { List shards = constructShardListForGraphA(); - List currentLeases = new ArrayList<>(); + List currentLeases = new ArrayList<>(); currentLeases.add(newLease("shardId-3")); currentLeases.add(newLease("shardId-4")); currentLeases.add(newLease("shardId-5")); - List newLeases = + List newLeases = ShardSyncer.determineNewLeasesToCreate(shards, currentLeases, INITIAL_POSITION_TRIM_HORIZON); Map expectedShardIdCheckpointMap = new HashMap<>(); expectedShardIdCheckpointMap.put("shardId-8", ExtendedSequenceNumber.TRIM_HORIZON); @@ -812,10 +816,10 @@ public class ShardSyncerTest { expectedShardIdCheckpointMap.put("shardId-1", ExtendedSequenceNumber.TRIM_HORIZON); assertEquals(expectedShardIdCheckpointMap.size(), newLeases.size()); - for (KinesisClientLease lease : newLeases) { + for (Lease lease : newLeases) { assertTrue("Unexpected lease: " + lease, - expectedShardIdCheckpointMap.containsKey(lease.getLeaseKey())); - assertEquals(expectedShardIdCheckpointMap.get(lease.getLeaseKey()), lease.getCheckpoint()); + expectedShardIdCheckpointMap.containsKey(lease.leaseKey())); + assertEquals(expectedShardIdCheckpointMap.get(lease.leaseKey()), lease.checkpoint()); } } @@ -832,13 +836,13 @@ public class ShardSyncerTest { @Test public final void testDetermineNewLeasesToCreateSplitMergeHorizon2() { List shards = constructShardListForGraphA(); - List currentLeases = new ArrayList<>(); + List currentLeases = new ArrayList<>(); currentLeases.add(newLease("shardId-4")); currentLeases.add(newLease("shardId-5")); currentLeases.add(newLease("shardId-7")); - List newLeases = + List newLeases = ShardSyncer.determineNewLeasesToCreate(shards, currentLeases, INITIAL_POSITION_TRIM_HORIZON); Map expectedShardIdCheckpointMap = new HashMap<>(); @@ -850,10 +854,10 @@ public class ShardSyncerTest { expectedShardIdCheckpointMap.put("shardId-1", ExtendedSequenceNumber.TRIM_HORIZON); assertEquals(expectedShardIdCheckpointMap.size(), newLeases.size()); - for (KinesisClientLease lease : newLeases) { + for (Lease lease : newLeases) { assertTrue("Unexpected lease: " + lease, - expectedShardIdCheckpointMap.containsKey(lease.getLeaseKey())); - assertEquals(expectedShardIdCheckpointMap.get(lease.getLeaseKey()), lease.getCheckpoint()); + expectedShardIdCheckpointMap.containsKey(lease.leaseKey())); + assertEquals(expectedShardIdCheckpointMap.get(lease.leaseKey()), lease.checkpoint()); } } @@ -866,8 +870,8 @@ public class ShardSyncerTest { @Test public final void testDetermineNewLeasesToCreateGraphBNoInitialLeasesTrim() { List shards = constructShardListForGraphB(); - List currentLeases = new ArrayList<>(); - List newLeases = + List currentLeases = new ArrayList<>(); + List newLeases = ShardSyncer.determineNewLeasesToCreate(shards, currentLeases, INITIAL_POSITION_TRIM_HORIZON); Map expectedShardIdCheckpointMap = new HashMap<>(); @@ -877,10 +881,10 @@ public class ShardSyncerTest { } assertEquals(expectedShardIdCheckpointMap.size(), newLeases.size()); - for (KinesisClientLease lease : newLeases) { + for (Lease lease : newLeases) { assertTrue("Unexpected lease: " + lease, - expectedShardIdCheckpointMap.containsKey(lease.getLeaseKey())); - assertEquals(expectedShardIdCheckpointMap.get(lease.getLeaseKey()), lease.getCheckpoint()); + expectedShardIdCheckpointMap.containsKey(lease.leaseKey())); + assertEquals(expectedShardIdCheckpointMap.get(lease.leaseKey()), lease.checkpoint()); } } @@ -897,14 +901,14 @@ public class ShardSyncerTest { @Test public final void testDetermineNewLeasesToCreateSplitMergeAtTimestamp1() { List shards = constructShardListForGraphA(); - List currentLeases = new ArrayList<>(); + List currentLeases = new ArrayList<>(); currentLeases.add(newLease("shardId-3")); currentLeases.add(newLease("shardId-4")); currentLeases.add(newLease("shardId-5")); - List newLeases = + List newLeases = ShardSyncer.determineNewLeasesToCreate(shards, currentLeases, INITIAL_POSITION_AT_TIMESTAMP); Map expectedShardIdCheckpointMap = new HashMap<>(); expectedShardIdCheckpointMap.put("shardId-8", ExtendedSequenceNumber.AT_TIMESTAMP); @@ -917,10 +921,10 @@ public class ShardSyncerTest { expectedShardIdCheckpointMap.put("shardId-1", ExtendedSequenceNumber.AT_TIMESTAMP); assertEquals(expectedShardIdCheckpointMap.size(), newLeases.size()); - for (KinesisClientLease lease : newLeases) { + for (Lease lease : newLeases) { assertTrue("Unexpected lease: " + lease, - expectedShardIdCheckpointMap.containsKey(lease.getLeaseKey())); - assertEquals(expectedShardIdCheckpointMap.get(lease.getLeaseKey()), lease.getCheckpoint()); + expectedShardIdCheckpointMap.containsKey(lease.leaseKey())); + assertEquals(expectedShardIdCheckpointMap.get(lease.leaseKey()), lease.checkpoint()); } } @@ -937,13 +941,13 @@ public class ShardSyncerTest { @Test public final void testDetermineNewLeasesToCreateSplitMergeAtTimestamp2() { List shards = constructShardListForGraphA(); - List currentLeases = new ArrayList<>(); + List currentLeases = new ArrayList<>(); currentLeases.add(newLease("shardId-4")); currentLeases.add(newLease("shardId-5")); currentLeases.add(newLease("shardId-7")); - List newLeases = + List newLeases = ShardSyncer.determineNewLeasesToCreate(shards, currentLeases, INITIAL_POSITION_AT_TIMESTAMP); Map expectedShardIdCheckpointMap = new HashMap<>(); expectedShardIdCheckpointMap.put("shardId-8", ExtendedSequenceNumber.AT_TIMESTAMP); @@ -954,10 +958,10 @@ public class ShardSyncerTest { expectedShardIdCheckpointMap.put("shardId-1", ExtendedSequenceNumber.AT_TIMESTAMP); assertEquals(expectedShardIdCheckpointMap.size(), newLeases.size()); - for (KinesisClientLease lease : newLeases) { + for (Lease lease : newLeases) { assertTrue("Unexpected lease: " + lease, - expectedShardIdCheckpointMap.containsKey(lease.getLeaseKey())); - assertEquals(expectedShardIdCheckpointMap.get(lease.getLeaseKey()), lease.getCheckpoint()); + expectedShardIdCheckpointMap.containsKey(lease.leaseKey())); + assertEquals(expectedShardIdCheckpointMap.get(lease.leaseKey()), lease.checkpoint()); } } @@ -969,8 +973,8 @@ public class ShardSyncerTest { @Test public final void testDetermineNewLeasesToCreateGraphBNoInitialLeasesAtTimestamp() { List shards = constructShardListForGraphB(); - List currentLeases = new ArrayList<>(); - List newLeases = + List currentLeases = new ArrayList<>(); + List newLeases = ShardSyncer.determineNewLeasesToCreate(shards, currentLeases, INITIAL_POSITION_AT_TIMESTAMP); Map expectedShardIdCheckpointMap = new HashMap<>(); @@ -980,10 +984,10 @@ public class ShardSyncerTest { } assertEquals(expectedShardIdCheckpointMap.size(), newLeases.size()); - for (KinesisClientLease lease : newLeases) { + for (Lease lease : newLeases) { assertTrue("Unexpected lease: " + lease, - expectedShardIdCheckpointMap.containsKey(lease.getLeaseKey())); - assertEquals(expectedShardIdCheckpointMap.get(lease.getLeaseKey()), lease.getCheckpoint()); + expectedShardIdCheckpointMap.containsKey(lease.leaseKey())); + assertEquals(expectedShardIdCheckpointMap.get(lease.leaseKey()), lease.checkpoint()); } } @@ -1112,7 +1116,7 @@ public class ShardSyncerTest { kinesisShards.put(shardId, ShardObjectHelper.newShard(shardId, null, null, null)); Set shardIdsOfCurrentLeases = new HashSet(); shardIdsOfCurrentLeases.add(shardId); - Map newLeaseMap = new HashMap(); + Map newLeaseMap = new HashMap(); Map memoizationContext = new HashMap<>(); assertTrue(ShardSyncer.checkIfDescendantAndAddNewLeasesForAncestors(shardId, INITIAL_POSITION_LATEST, shardIdsOfCurrentLeases, @@ -1128,7 +1132,7 @@ public class ShardSyncerTest { @Test public final void testCheckIfDescendantAndAddNewLeasesForAncestors2P2ANotDescendant() { Set shardIdsOfCurrentLeases = new HashSet(); - Map newLeaseMap = new HashMap(); + Map newLeaseMap = new HashMap(); Map kinesisShards = new HashMap(); String parentShardId = "shardId-parent"; @@ -1155,7 +1159,7 @@ public class ShardSyncerTest { @Test public final void testCheckIfDescendantAndAddNewLeasesForAncestors2P2A1PDescendant() { Set shardIdsOfCurrentLeases = new HashSet(); - Map newLeaseMap = new HashMap(); + Map newLeaseMap = new HashMap(); Map kinesisShards = new HashMap(); String parentShardId = "shardId-parent"; @@ -1177,8 +1181,8 @@ public class ShardSyncerTest { memoizationContext)); assertEquals(1, newLeaseMap.size()); assertTrue(newLeaseMap.containsKey(adjacentParentShardId)); - KinesisClientLease adjacentParentLease = newLeaseMap.get(adjacentParentShardId); - assertEquals(ExtendedSequenceNumber.LATEST, adjacentParentLease.getCheckpoint()); + Lease adjacentParentLease = newLeaseMap.get(adjacentParentShardId); + assertEquals(ExtendedSequenceNumber.LATEST, adjacentParentLease.checkpoint()); } /** @@ -1288,10 +1292,10 @@ public class ShardSyncerTest { shard.setParentShardId(parentShardId); shard.setAdjacentParentShardId(adjacentParentShardId); - KinesisClientLease lease = ShardSyncer.newKCLLease(shard); - assertEquals(shardId, lease.getLeaseKey()); - assertNull(lease.getCheckpoint()); - Set parentIds = lease.getParentShardIds(); + Lease lease = ShardSyncer.newKCLLease(shard); + assertEquals(shardId, lease.leaseKey()); + assertNull(lease.checkpoint()); + Set parentIds = lease.parentShardIds(); assertEquals(2, parentIds.size()); assertTrue(parentIds.contains(parentShardId)); assertTrue(parentIds.contains(adjacentParentShardId)); @@ -1364,11 +1368,11 @@ public class ShardSyncerTest { String parentShardId = "shardId-0000"; String adjacentParentShardId = "shardId-0001"; String shardId = "shardId-0002"; - KinesisClientLease lease = newLease(shardId); + Lease lease = newLease(shardId); List parentShardIds = new ArrayList<>(); parentShardIds.add(parentShardId); parentShardIds.add(adjacentParentShardId); - lease.setParentShardIds(parentShardIds); + lease.parentShardIds(parentShardIds); Set currentKinesisShardIds = new HashSet<>(); currentKinesisShardIds.add(shardId); @@ -1401,11 +1405,11 @@ public class ShardSyncerTest { String parentShardId = "shardId-0000"; String adjacentParentShardId = "shardId-0001"; String shardId = "shardId-0002"; - KinesisClientLease lease = newLease(shardId); + Lease lease = newLease(shardId); List parentShardIds = new ArrayList<>(); parentShardIds.add(parentShardId); parentShardIds.add(adjacentParentShardId); - lease.setParentShardIds(parentShardIds); + lease.parentShardIds(parentShardIds); Set currentKinesisShardIds = new HashSet<>(); currentKinesisShardIds.add(parentShardId); @@ -1422,11 +1426,11 @@ public class ShardSyncerTest { String parentShardId = "shardId-0000"; String adjacentParentShardId = "shardId-0001"; String shardId = "shardId-0002"; - KinesisClientLease lease = newLease(shardId); + Lease lease = newLease(shardId); List parentShardIds = new ArrayList<>(); parentShardIds.add(parentShardId); parentShardIds.add(adjacentParentShardId); - lease.setParentShardIds(parentShardIds); + lease.parentShardIds(parentShardIds); Set currentKinesisShardIds = new HashSet<>(); currentKinesisShardIds.add(adjacentParentShardId); @@ -1444,68 +1448,68 @@ public class ShardSyncerTest { public final void testCleanupLeaseForClosedShard() throws DependencyException, InvalidStateException, ProvisionedThroughputException { String closedShardId = "shardId-2"; - KinesisClientLease leaseForClosedShard = newLease(closedShardId); - leaseForClosedShard.setCheckpoint(new ExtendedSequenceNumber("1234")); - dynamoDBLeaseManager.createLeaseIfNotExists(leaseForClosedShard); + Lease leaseForClosedShard = newLease(closedShardId); + leaseForClosedShard.checkpoint(new ExtendedSequenceNumber("1234")); + dynamoDBLeaseRefresher.createLeaseIfNotExists(leaseForClosedShard); Set childShardIds = new HashSet<>(); - List trackedLeases = new ArrayList<>(); + List trackedLeases = new ArrayList<>(); Set parentShardIds = new HashSet<>(); parentShardIds.add(closedShardId); String childShardId1 = "shardId-5"; - KinesisClientLease childLease1 = newLease(childShardId1); - childLease1.setParentShardIds(parentShardIds); - childLease1.setCheckpoint(ExtendedSequenceNumber.TRIM_HORIZON); + Lease childLease1 = newLease(childShardId1); + childLease1.parentShardIds(parentShardIds); + childLease1.checkpoint(ExtendedSequenceNumber.TRIM_HORIZON); String childShardId2 = "shardId-7"; - KinesisClientLease childLease2 = newLease(childShardId2); - childLease2.setParentShardIds(parentShardIds); - childLease2.setCheckpoint(ExtendedSequenceNumber.TRIM_HORIZON); - Map trackedLeaseMap = ShardSyncer.constructShardIdToKCLLeaseMap(trackedLeases); + Lease childLease2 = newLease(childShardId2); + childLease2.parentShardIds(parentShardIds); + childLease2.checkpoint(ExtendedSequenceNumber.TRIM_HORIZON); + Map trackedLeaseMap = ShardSyncer.constructShardIdToKCLLeaseMap(trackedLeases); // empty list of leases - ShardSyncer.cleanupLeaseForClosedShard(closedShardId, childShardIds, trackedLeaseMap, dynamoDBLeaseManager); - assertNotNull(dynamoDBLeaseManager.getLease(closedShardId)); + ShardSyncer.cleanupLeaseForClosedShard(closedShardId, childShardIds, trackedLeaseMap, dynamoDBLeaseRefresher); + assertNotNull(dynamoDBLeaseRefresher.getLease(closedShardId)); // closed shard has not been fully processed yet (checkpoint != SHARD_END) trackedLeases.add(leaseForClosedShard); trackedLeaseMap = ShardSyncer.constructShardIdToKCLLeaseMap(trackedLeases); - ShardSyncer.cleanupLeaseForClosedShard(closedShardId, childShardIds, trackedLeaseMap, dynamoDBLeaseManager); - assertNotNull(dynamoDBLeaseManager.getLease(closedShardId)); + ShardSyncer.cleanupLeaseForClosedShard(closedShardId, childShardIds, trackedLeaseMap, dynamoDBLeaseRefresher); + assertNotNull(dynamoDBLeaseRefresher.getLease(closedShardId)); // closed shard has been fully processed yet (checkpoint == SHARD_END) - leaseForClosedShard.setCheckpoint(ExtendedSequenceNumber.SHARD_END); - dynamoDBLeaseManager.updateLease(leaseForClosedShard); - ShardSyncer.cleanupLeaseForClosedShard(closedShardId, childShardIds, trackedLeaseMap, dynamoDBLeaseManager); - assertNull(dynamoDBLeaseManager.getLease(closedShardId)); + leaseForClosedShard.checkpoint(ExtendedSequenceNumber.SHARD_END); + dynamoDBLeaseRefresher.updateLease(leaseForClosedShard); + ShardSyncer.cleanupLeaseForClosedShard(closedShardId, childShardIds, trackedLeaseMap, dynamoDBLeaseRefresher); + assertNull(dynamoDBLeaseRefresher.getLease(closedShardId)); // lease for only one child exists childShardIds.add(childShardId1); childShardIds.add(childShardId2); - dynamoDBLeaseManager.createLeaseIfNotExists(leaseForClosedShard); - dynamoDBLeaseManager.createLeaseIfNotExists(childLease1); + dynamoDBLeaseRefresher.createLeaseIfNotExists(leaseForClosedShard); + dynamoDBLeaseRefresher.createLeaseIfNotExists(childLease1); trackedLeases.add(childLease1); trackedLeaseMap = ShardSyncer.constructShardIdToKCLLeaseMap(trackedLeases); - ShardSyncer.cleanupLeaseForClosedShard(closedShardId, childShardIds, trackedLeaseMap, dynamoDBLeaseManager); - assertNotNull(dynamoDBLeaseManager.getLease(closedShardId)); + ShardSyncer.cleanupLeaseForClosedShard(closedShardId, childShardIds, trackedLeaseMap, dynamoDBLeaseRefresher); + assertNotNull(dynamoDBLeaseRefresher.getLease(closedShardId)); // leases for both children exists, but they are both at TRIM_HORIZON - dynamoDBLeaseManager.createLeaseIfNotExists(childLease2); + dynamoDBLeaseRefresher.createLeaseIfNotExists(childLease2); trackedLeases.add(childLease2); trackedLeaseMap = ShardSyncer.constructShardIdToKCLLeaseMap(trackedLeases); - ShardSyncer.cleanupLeaseForClosedShard(closedShardId, childShardIds, trackedLeaseMap, dynamoDBLeaseManager); - assertNotNull(dynamoDBLeaseManager.getLease(closedShardId)); + ShardSyncer.cleanupLeaseForClosedShard(closedShardId, childShardIds, trackedLeaseMap, dynamoDBLeaseRefresher); + assertNotNull(dynamoDBLeaseRefresher.getLease(closedShardId)); // leases for both children exists, one is at TRIM_HORIZON - childLease1.setCheckpoint(new ExtendedSequenceNumber("34890")); - dynamoDBLeaseManager.updateLease(childLease1); - ShardSyncer.cleanupLeaseForClosedShard(closedShardId, childShardIds, trackedLeaseMap, dynamoDBLeaseManager); - assertNotNull(dynamoDBLeaseManager.getLease(closedShardId)); + childLease1.checkpoint(new ExtendedSequenceNumber("34890")); + dynamoDBLeaseRefresher.updateLease(childLease1); + ShardSyncer.cleanupLeaseForClosedShard(closedShardId, childShardIds, trackedLeaseMap, dynamoDBLeaseRefresher); + assertNotNull(dynamoDBLeaseRefresher.getLease(closedShardId)); // leases for both children exists, NONE of them are at TRIM_HORIZON - childLease2.setCheckpoint(new ExtendedSequenceNumber("43789")); - dynamoDBLeaseManager.updateLease(childLease2); - ShardSyncer.cleanupLeaseForClosedShard(closedShardId, childShardIds, trackedLeaseMap, dynamoDBLeaseManager); - assertNull(dynamoDBLeaseManager.getLease(closedShardId)); + childLease2.checkpoint(new ExtendedSequenceNumber("43789")); + dynamoDBLeaseRefresher.updateLease(childLease2); + ShardSyncer.cleanupLeaseForClosedShard(closedShardId, childShardIds, trackedLeaseMap, dynamoDBLeaseRefresher); + assertNull(dynamoDBLeaseRefresher.getLease(closedShardId)); } /** @@ -1675,9 +1679,9 @@ public class ShardSyncerTest { * @param shardId * @return */ - private KinesisClientLease newLease(String shardId) { - KinesisClientLease lease = new KinesisClientLease(); - lease.setLeaseKey(shardId); + private Lease newLease(String shardId) { + Lease lease = new Lease(); + lease.leaseKey(shardId); return lease; } diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/TestHarnessBuilder.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/TestHarnessBuilder.java deleted file mode 100644 index 8e4a45cd..00000000 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/TestHarnessBuilder.java +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/asl/ - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ -package software.amazon.kinesis.leases; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Callable; - -import org.junit.Assert; - -import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; -import software.amazon.kinesis.leases.exceptions.DependencyException; -import software.amazon.kinesis.leases.exceptions.InvalidStateException; -import software.amazon.kinesis.leases.exceptions.LeasingException; - -public class TestHarnessBuilder { - - private long currentTimeNanos; - - private Map leases = new HashMap(); - private KinesisClientDynamoDBLeaseManager leaseManager; - private Map originalLeases = new HashMap<>(); - - private Callable timeProvider = new Callable() { - - @Override - public Long call() throws Exception { - return currentTimeNanos; - } - - }; - - public TestHarnessBuilder(KinesisClientDynamoDBLeaseManager leaseManager) { - this.leaseManager = leaseManager; - } - - public TestHarnessBuilder withLease(String shardId) { - return withLease(shardId, "leaseOwner"); - } - - public TestHarnessBuilder withLease(String shardId, String owner) { - KinesisClientLease lease = createLease(shardId, owner); - KinesisClientLease originalLease = createLease(shardId, owner); - - leases.put(shardId, lease); - originalLeases.put(shardId, originalLease); - return this; - } - - private KinesisClientLease createLease(String shardId, String owner) { - KinesisClientLease lease = new KinesisClientLease(); - lease.setCheckpoint(new ExtendedSequenceNumber("checkpoint")); - lease.setOwnerSwitchesSinceCheckpoint(0L); - lease.setLeaseCounter(0L); - lease.setLeaseOwner(owner); - lease.setParentShardIds(Collections.singleton("parentShardId")); - lease.setLeaseKey(shardId); - - return lease; - } - - public Map build() throws LeasingException { - for (KinesisClientLease lease : leases.values()) { - leaseManager.createLeaseIfNotExists(lease); - if (lease.getLeaseOwner() != null) { - lease.setLastCounterIncrementNanos(System.nanoTime()); - } - } - - currentTimeNanos = System.nanoTime(); - - return leases; - } - - public void passTime(long millis) { - currentTimeNanos += millis * 1000000; - } - - public Map takeMutateAssert(DynamoDBLeaseTaker taker, int numToTake) - throws LeasingException { - Map result = taker.takeLeases(timeProvider); - Assert.assertEquals(numToTake, result.size()); - - for (KinesisClientLease actual : result.values()) { - KinesisClientLease original = leases.get(actual.getLeaseKey()); - Assert.assertNotNull(original); - - mutateAssert(taker.getWorkerIdentifier(), original, actual); - } - - return result; - } - - public Map takeMutateAssert(DynamoDBLeaseTaker taker, String... takenShardIds) - throws LeasingException { - Map result = taker.takeLeases(timeProvider); - Assert.assertEquals(takenShardIds.length, result.size()); - - for (String shardId : takenShardIds) { - KinesisClientLease original = leases.get(shardId); - Assert.assertNotNull(original); - - KinesisClientLease actual = result.get(shardId); - Assert.assertNotNull(actual); - - mutateAssert(taker.getWorkerIdentifier(), original, actual); - } - - return result; - } - - private void mutateAssert(String newWorkerIdentifier, KinesisClientLease original, KinesisClientLease actual) { - original.setLeaseCounter(original.getLeaseCounter() + 1); - if (original.getLeaseOwner() != null && !newWorkerIdentifier.equals(original.getLeaseOwner())) { - original.setOwnerSwitchesSinceCheckpoint(original.getOwnerSwitchesSinceCheckpoint() + 1); - } - original.setLeaseOwner(newWorkerIdentifier); - - Assert.assertEquals(original, actual); // Assert the contents of the lease - } - - public void addLeasesToRenew(LeaseRenewer renewer, String... shardIds) - throws DependencyException, InvalidStateException { - List leasesToRenew = new ArrayList(); - - for (String shardId : shardIds) { - KinesisClientLease lease = leases.get(shardId); - Assert.assertNotNull(lease); - leasesToRenew.add(lease); - } - - renewer.addLeasesToRenew(leasesToRenew); - } - - public Map renewMutateAssert(LeaseRenewer renewer, String... renewedShardIds) - throws DependencyException, InvalidStateException { - renewer.renewLeases(); - - Map heldLeases = renewer.getCurrentlyHeldLeases(); - Assert.assertEquals(renewedShardIds.length, heldLeases.size()); - - for (String shardId : renewedShardIds) { - KinesisClientLease original = originalLeases.get(shardId); - Assert.assertNotNull(original); - - KinesisClientLease actual = heldLeases.get(shardId); - Assert.assertNotNull(actual); - - original.setLeaseCounter(original.getLeaseCounter() + 1); - Assert.assertEquals(original, actual); - } - - return heldLeases; - } - - public void renewAllLeases() throws LeasingException { - for (KinesisClientLease lease : leases.values()) { - leaseManager.renewLease(lease); - } - } -} diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/KinesisClientLibLeaseCoordinatorIntegrationTest.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseCoordinatorIntegrationTest.java similarity index 60% rename from amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/KinesisClientLibLeaseCoordinatorIntegrationTest.java rename to amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseCoordinatorIntegrationTest.java index 6ddd1d93..ca2e82bd 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/KinesisClientLibLeaseCoordinatorIntegrationTest.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseCoordinatorIntegrationTest.java @@ -12,7 +12,7 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package software.amazon.kinesis.leases; +package software.amazon.kinesis.leases.dynamodb; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -36,6 +36,9 @@ import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; import software.amazon.kinesis.checkpoint.DynamoDBCheckpointer; +import software.amazon.kinesis.leases.Lease; +import software.amazon.kinesis.leases.LeaseCoordinator; +import software.amazon.kinesis.leases.LeaseRenewer; import software.amazon.kinesis.leases.exceptions.DependencyException; import software.amazon.kinesis.leases.exceptions.InvalidStateException; import software.amazon.kinesis.leases.exceptions.LeasingException; @@ -45,35 +48,35 @@ import software.amazon.kinesis.metrics.NullMetricsFactory; import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; @RunWith(MockitoJUnitRunner.class) -public class KinesisClientLibLeaseCoordinatorIntegrationTest { - private static final String TABLE_NAME = KinesisClientLibLeaseCoordinatorIntegrationTest.class.getSimpleName(); +public class DynamoDBLeaseCoordinatorIntegrationTest { + private static final String TABLE_NAME = DynamoDBLeaseCoordinatorIntegrationTest.class.getSimpleName(); private static final String WORKER_ID = UUID.randomUUID().toString(); private static final long LEASE_DURATION_MILLIS = 5000L; private static final long EPSILON_MILLIS = 25L; private static final int MAX_LEASES_FOR_WORKER = Integer.MAX_VALUE; private static final int MAX_LEASES_TO_STEAL_AT_ONE_TIME = 1; private static final int MAX_LEASE_RENEWER_THREAD_COUNT = 20; - private static KinesisClientDynamoDBLeaseManager leaseManager; + private static DynamoDBLeaseRefresher leaseRefresher; private static DynamoDBCheckpointer dynamoDBCheckpointer; - private KinesisClientLibLeaseCoordinator coordinator; + private LeaseCoordinator coordinator; private final String leaseKey = "shd-1"; private final IMetricsFactory metricsFactory = new NullMetricsFactory(); @Before public void setUp() throws ProvisionedThroughputException, DependencyException, InvalidStateException { final boolean useConsistentReads = true; - if (leaseManager == null) { + if (leaseRefresher == null) { AmazonDynamoDBClient ddb = new AmazonDynamoDBClient(new DefaultAWSCredentialsProviderChain()); - leaseManager = - new KinesisClientDynamoDBLeaseManager(TABLE_NAME, ddb, useConsistentReads); + leaseRefresher = + new DynamoDBLeaseRefresher(TABLE_NAME, ddb, new DynamoDBLeaseSerializer(), useConsistentReads); } - leaseManager.createLeaseTableIfNotExists(10L, 10L); - leaseManager.deleteAll(); - coordinator = new KinesisClientLibLeaseCoordinator(leaseManager, WORKER_ID, LEASE_DURATION_MILLIS, + leaseRefresher.createLeaseTableIfNotExists(10L, 10L); + leaseRefresher.deleteAll(); + coordinator = new DynamoDBLeaseCoordinator(leaseRefresher, WORKER_ID, LEASE_DURATION_MILLIS, EPSILON_MILLIS, MAX_LEASES_FOR_WORKER, MAX_LEASES_TO_STEAL_AT_ONE_TIME, MAX_LEASE_RENEWER_THREAD_COUNT, metricsFactory); - dynamoDBCheckpointer = new DynamoDBCheckpointer(coordinator, leaseManager, metricsFactory); + dynamoDBCheckpointer = new DynamoDBCheckpointer(coordinator, leaseRefresher, metricsFactory); coordinator.start(); } @@ -86,14 +89,14 @@ public class KinesisClientLibLeaseCoordinatorIntegrationTest { TestHarnessBuilder builder = new TestHarnessBuilder(); builder.withLease(leaseKey, null).build(); - // Run the taker and renewer in-between getting the Lease object and calling setCheckpoint + // Run the taker and renewer in-between getting the Lease object and calling checkpoint coordinator.runLeaseTaker(); coordinator.runLeaseRenewer(); - KinesisClientLease lease = coordinator.getCurrentlyHeldLease(leaseKey); + Lease lease = coordinator.getCurrentlyHeldLease(leaseKey); if (lease == null) { - List leases = leaseManager.listLeases(); - for (KinesisClientLease kinesisClientLease : leases) { + List leases = leaseRefresher.listLeases(); + for (Lease kinesisClientLease : leases) { System.out.println(kinesisClientLease); } } @@ -101,13 +104,13 @@ public class KinesisClientLibLeaseCoordinatorIntegrationTest { assertNotNull(lease); ExtendedSequenceNumber newCheckpoint = new ExtendedSequenceNumber("newCheckpoint"); // lease's leaseCounter is wrong at this point, but it shouldn't matter. - assertTrue(dynamoDBCheckpointer.setCheckpoint(lease.getLeaseKey(), newCheckpoint, lease.getConcurrencyToken())); + assertTrue(dynamoDBCheckpointer.setCheckpoint(lease.leaseKey(), newCheckpoint, lease.concurrencyToken())); - Lease fromDynamo = leaseManager.getLease(lease.getLeaseKey()); + Lease fromDynamo = leaseRefresher.getLease(lease.leaseKey()); - lease.setLeaseCounter(lease.getLeaseCounter() + 1); - lease.setCheckpoint(newCheckpoint); - lease.setLeaseOwner(coordinator.getWorkerIdentifier()); + lease.leaseCounter(lease.leaseCounter() + 1); + lease.checkpoint(newCheckpoint); + lease.leaseOwner(coordinator.workerIdentifier()); assertEquals(lease, fromDynamo); } @@ -121,19 +124,19 @@ public class KinesisClientLibLeaseCoordinatorIntegrationTest { coordinator.runLeaseTaker(); coordinator.runLeaseRenewer(); - KinesisClientLease lease = coordinator.getCurrentlyHeldLease(leaseKey); + Lease lease = coordinator.getCurrentlyHeldLease(leaseKey); assertNotNull(lease); - leaseManager.renewLease(coordinator.getCurrentlyHeldLease(leaseKey)); + leaseRefresher.renewLease(coordinator.getCurrentlyHeldLease(leaseKey)); ExtendedSequenceNumber newCheckpoint = new ExtendedSequenceNumber("newCheckpoint"); - assertFalse(dynamoDBCheckpointer.setCheckpoint(lease.getLeaseKey(), newCheckpoint, lease.getConcurrencyToken())); + assertFalse(dynamoDBCheckpointer.setCheckpoint(lease.leaseKey(), newCheckpoint, lease.concurrencyToken())); - Lease fromDynamo = leaseManager.getLease(lease.getLeaseKey()); + Lease fromDynamo = leaseRefresher.getLease(lease.leaseKey()); - lease.setLeaseCounter(lease.getLeaseCounter() + 1); + lease.leaseCounter(lease.leaseCounter() + 1); // Counter and owner changed, but checkpoint did not. - lease.setLeaseOwner(coordinator.getWorkerIdentifier()); + lease.leaseOwner(coordinator.workerIdentifier()); assertEquals(lease, fromDynamo); } @@ -147,17 +150,17 @@ public class KinesisClientLibLeaseCoordinatorIntegrationTest { coordinator.runLeaseTaker(); coordinator.runLeaseRenewer(); - KinesisClientLease lease = coordinator.getCurrentlyHeldLease(leaseKey); + Lease lease = coordinator.getCurrentlyHeldLease(leaseKey); assertNotNull(lease); ExtendedSequenceNumber newCheckpoint = new ExtendedSequenceNumber("newCheckpoint"); - assertFalse(dynamoDBCheckpointer.setCheckpoint(lease.getLeaseKey(), newCheckpoint, UUID.randomUUID())); + assertFalse(dynamoDBCheckpointer.setCheckpoint(lease.leaseKey(), newCheckpoint, UUID.randomUUID())); - Lease fromDynamo = leaseManager.getLease(lease.getLeaseKey()); + Lease fromDynamo = leaseRefresher.getLease(lease.leaseKey()); // Owner should be the only thing that changed. - lease.setLeaseOwner(coordinator.getWorkerIdentifier()); + lease.leaseOwner(coordinator.workerIdentifier()); assertEquals(lease, fromDynamo); } @@ -165,7 +168,7 @@ public class KinesisClientLibLeaseCoordinatorIntegrationTest { private long currentTimeNanos; - private Map leases = new HashMap(); + private Map leases = new HashMap(); private Callable timeProvider = new Callable() { @@ -181,23 +184,23 @@ public class KinesisClientLibLeaseCoordinatorIntegrationTest { } public TestHarnessBuilder withLease(String shardId, String owner) { - KinesisClientLease lease = new KinesisClientLease(); - lease.setCheckpoint(new ExtendedSequenceNumber("checkpoint")); - lease.setOwnerSwitchesSinceCheckpoint(0L); - lease.setLeaseCounter(0L); - lease.setLeaseOwner(owner); - lease.setParentShardIds(Collections.singleton("parentShardId")); - lease.setLeaseKey(shardId); + Lease lease = new Lease(); + lease.checkpoint(new ExtendedSequenceNumber("checkpoint")); + lease.ownerSwitchesSinceCheckpoint(0L); + lease.leaseCounter(0L); + lease.leaseOwner(owner); + lease.parentShardIds(Collections.singleton("parentShardId")); + lease.leaseKey(shardId); leases.put(shardId, lease); return this; } - public Map build() throws LeasingException { - for (KinesisClientLease lease : leases.values()) { - leaseManager.createLeaseIfNotExists(lease); - if (lease.getLeaseOwner() != null) { - lease.setLastCounterIncrementNanos(System.nanoTime()); + public Map build() throws LeasingException { + for (Lease lease : leases.values()) { + leaseRefresher.createLeaseIfNotExists(lease); + if (lease.leaseOwner() != null) { + lease.lastCounterIncrementNanos(System.nanoTime()); } } @@ -210,22 +213,22 @@ public class KinesisClientLibLeaseCoordinatorIntegrationTest { currentTimeNanos += millis * 1000000; } - private void mutateAssert(String newWorkerIdentifier, KinesisClientLease original, KinesisClientLease actual) { - original.setLeaseCounter(original.getLeaseCounter() + 1); - if (original.getLeaseOwner() != null && !newWorkerIdentifier.equals(original.getLeaseOwner())) { - original.setOwnerSwitchesSinceCheckpoint(original.getOwnerSwitchesSinceCheckpoint() + 1); + private void mutateAssert(String newWorkerIdentifier, Lease original, Lease actual) { + original.leaseCounter(original.leaseCounter() + 1); + if (original.leaseOwner() != null && !newWorkerIdentifier.equals(original.leaseOwner())) { + original.ownerSwitchesSinceCheckpoint(original.ownerSwitchesSinceCheckpoint() + 1); } - original.setLeaseOwner(newWorkerIdentifier); + original.leaseOwner(newWorkerIdentifier); assertEquals(original, actual); // Assert the contents of the lease } - public void addLeasesToRenew(LeaseRenewer renewer, String... shardIds) + public void addLeasesToRenew(LeaseRenewer renewer, String... shardIds) throws DependencyException, InvalidStateException { - List leasesToRenew = new ArrayList(); + List leasesToRenew = new ArrayList<>(); for (String shardId : shardIds) { - KinesisClientLease lease = leases.get(shardId); + Lease lease = leases.get(shardId); assertNotNull(lease); leasesToRenew.add(lease); } @@ -233,21 +236,21 @@ public class KinesisClientLibLeaseCoordinatorIntegrationTest { renewer.addLeasesToRenew(leasesToRenew); } - public Map renewMutateAssert(LeaseRenewer renewer, + public Map renewMutateAssert(LeaseRenewer renewer, String... renewedShardIds) throws DependencyException, InvalidStateException { renewer.renewLeases(); - Map heldLeases = renewer.getCurrentlyHeldLeases(); + Map heldLeases = renewer.getCurrentlyHeldLeases(); assertEquals(renewedShardIds.length, heldLeases.size()); for (String shardId : renewedShardIds) { - KinesisClientLease original = leases.get(shardId); + Lease original = leases.get(shardId); assertNotNull(original); - KinesisClientLease actual = heldLeases.get(shardId); + Lease actual = heldLeases.get(shardId); assertNotNull(actual); - original.setLeaseCounter(original.getLeaseCounter() + 1); + original.leaseCounter(original.leaseCounter() + 1); assertEquals(original, actual); } @@ -255,8 +258,8 @@ public class KinesisClientLibLeaseCoordinatorIntegrationTest { } public void renewAllLeases() throws LeasingException { - for (KinesisClientLease lease : leases.values()) { - leaseManager.renewLease(lease); + for (Lease lease : leases.values()) { + leaseRefresher.renewLease(lease); } } } diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/KinesisClientLibLeaseCoordinatorTest.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseCoordinatorTest.java similarity index 81% rename from amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/KinesisClientLibLeaseCoordinatorTest.java rename to amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseCoordinatorTest.java index 733c13e7..15996db7 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/KinesisClientLibLeaseCoordinatorTest.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseCoordinatorTest.java @@ -12,7 +12,7 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package software.amazon.kinesis.leases; +package software.amazon.kinesis.leases.dynamodb; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.verify; @@ -24,29 +24,31 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; import com.amazonaws.services.kinesis.clientlibrary.exceptions.KinesisClientLibException; import com.amazonaws.services.kinesis.clientlibrary.exceptions.ShutdownException; -import org.mockito.runners.MockitoJUnitRunner; + import software.amazon.kinesis.checkpoint.DynamoDBCheckpointer; -import software.amazon.kinesis.metrics.IMetricsFactory; -import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; +import software.amazon.kinesis.leases.Lease; +import software.amazon.kinesis.leases.LeaseCoordinator; +import software.amazon.kinesis.leases.LeaseRefresher; import software.amazon.kinesis.leases.exceptions.DependencyException; import software.amazon.kinesis.leases.exceptions.InvalidStateException; import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException; +import software.amazon.kinesis.metrics.IMetricsFactory; +import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; @RunWith(MockitoJUnitRunner.class) -public class KinesisClientLibLeaseCoordinatorTest { +public class DynamoDBLeaseCoordinatorTest { private static final String SHARD_ID = "shardId-test"; - private static final String WORK_ID = "workId-test"; - private static final long TEST_LONG = 1000L; private static final ExtendedSequenceNumber TEST_CHKPT = new ExtendedSequenceNumber("string-test"); private static final UUID TEST_UUID = UUID.randomUUID(); @Mock - private LeaseManager leaseManager; + private LeaseRefresher leaseRefresher; @Mock - private LeaseCoordinator leaseCoordinator; + private LeaseCoordinator leaseCoordinator; @Mock private IMetricsFactory metricsFactory; @@ -55,18 +57,18 @@ public class KinesisClientLibLeaseCoordinatorTest { @SuppressWarnings("unchecked") @Before public void setUpLeaseCoordinator() throws ProvisionedThroughputException, DependencyException { - dynamoDBCheckpointer = new DynamoDBCheckpointer(leaseCoordinator, leaseManager, metricsFactory); + dynamoDBCheckpointer = new DynamoDBCheckpointer(leaseCoordinator, leaseRefresher, metricsFactory); } @Test(expected = ShutdownException.class) public void testSetCheckpointWithUnownedShardId() throws KinesisClientLibException, DependencyException, InvalidStateException, ProvisionedThroughputException { - final KinesisClientLease lease = new KinesisClientLease(); - when(leaseManager.getLease(eq(SHARD_ID))).thenReturn(lease); + final Lease lease = new Lease(); + when(leaseRefresher.getLease(eq(SHARD_ID))).thenReturn(lease); when(leaseCoordinator.updateLease(eq(lease), eq(TEST_UUID))).thenReturn(false); dynamoDBCheckpointer.setCheckpoint(SHARD_ID, TEST_CHKPT, TEST_UUID.toString()); - verify(leaseManager).getLease(eq(SHARD_ID)); + verify(leaseRefresher).getLease(eq(SHARD_ID)); verify(leaseCoordinator).updateLease(eq(lease), eq(TEST_UUID)); } @@ -74,7 +76,7 @@ public class KinesisClientLibLeaseCoordinatorTest { // public void testWaitLeaseTableTimeout() // throws DependencyException, ProvisionedThroughputException, IllegalStateException { // Set mock lease manager to return false in waiting -// doReturn(false).when(leaseManager).waitUntilLeaseTableExists(anyLong(), anyLong()); +// doReturn(false).when(leaseRefresher).waitUntilLeaseTableExists(anyLong(), anyLong()); // leaseCoordinator.initialize(); // } } diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/DynamoDBLeaseManagerIntegrationTest.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseRefresherIntegrationTest.java similarity index 52% rename from amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/DynamoDBLeaseManagerIntegrationTest.java rename to amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseRefresherIntegrationTest.java index 86473392..9d156944 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/DynamoDBLeaseManagerIntegrationTest.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseRefresherIntegrationTest.java @@ -12,27 +12,37 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package software.amazon.kinesis.leases; +package software.amazon.kinesis.leases.dynamodb; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.util.Collection; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; -import junit.framework.Assert; - import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; +import software.amazon.kinesis.leases.Lease; +import software.amazon.kinesis.leases.LeaseIntegrationTest; +import software.amazon.kinesis.leases.LeaseSerializer; import software.amazon.kinesis.leases.exceptions.LeasingException; -public class DynamoDBLeaseManagerIntegrationTest extends LeaseIntegrationTest { - +public class DynamoDBLeaseRefresherIntegrationTest extends LeaseIntegrationTest { /** * Test listLeases when no records are present. */ @Test public void testListNoRecords() throws LeasingException { - List leases = leaseManager.listLeases(); - Assert.assertTrue(leases.isEmpty()); + List leases = leaseRefresher.listLeases(); + assertTrue(leases.isEmpty()); } /** @@ -40,7 +50,7 @@ public class DynamoDBLeaseManagerIntegrationTest extends LeaseIntegrationTest { */ @Test public void testListWithRecords() throws LeasingException { - TestHarnessBuilder builder = new TestHarnessBuilder(leaseManager); + TestHarnessBuilder builder = new TestHarnessBuilder(leaseRefresher); int numRecordsToPut = 10; @@ -48,16 +58,16 @@ public class DynamoDBLeaseManagerIntegrationTest extends LeaseIntegrationTest { builder.withLease(Integer.toString(i)); } - Collection expected = builder.build().values(); + Collection expected = builder.build().values(); // The / 3 here ensures that we will test Dynamo's paging mechanics. - List actual = leaseManager.list(numRecordsToPut / 3); + List actual = leaseRefresher.list(numRecordsToPut / 3); - for (KinesisClientLease lease : actual) { - Assert.assertNotNull(expected.remove(lease)); + for (Lease lease : actual) { + assertNotNull(expected.remove(lease)); } - Assert.assertTrue(expected.isEmpty()); + assertTrue(expected.isEmpty()); } /** @@ -65,53 +75,53 @@ public class DynamoDBLeaseManagerIntegrationTest extends LeaseIntegrationTest { */ @Test public void testGetLease() throws LeasingException { - TestHarnessBuilder builder = new TestHarnessBuilder(leaseManager); + TestHarnessBuilder builder = new TestHarnessBuilder(leaseRefresher); Lease expected = builder.withLease("1").build().get("1"); - Lease actual = leaseManager.getLease(expected.getLeaseKey()); - Assert.assertEquals(expected, actual); + Lease actual = leaseRefresher.getLease(expected.leaseKey()); + assertEquals(expected, actual); } /** - * Tests leaseManager.get() when the looked-for record is absent. + * Tests leaseRefresher.get() when the looked-for record is absent. */ @Test public void testGetNull() throws LeasingException { - Lease actual = leaseManager.getLease("bogusShardId"); - Assert.assertNull(actual); + Lease actual = leaseRefresher.getLease("bogusShardId"); + assertNull(actual); } /** - * Tests leaseManager.holdLease's success scenario. + * Tests leaseRefresher.holdLease's success scenario. */ @Test public void testRenewLease() throws LeasingException { - TestHarnessBuilder builder = new TestHarnessBuilder(leaseManager); - KinesisClientLease lease = builder.withLease("1").build().get("1"); - Long originalLeaseCounter = lease.getLeaseCounter(); + TestHarnessBuilder builder = new TestHarnessBuilder(leaseRefresher); + Lease lease = builder.withLease("1").build().get("1"); + Long originalLeaseCounter = lease.leaseCounter(); - leaseManager.renewLease(lease); - Assert.assertTrue(originalLeaseCounter + 1 == lease.getLeaseCounter()); + leaseRefresher.renewLease(lease); + assertTrue(originalLeaseCounter + 1 == lease.leaseCounter()); - Lease fromDynamo = leaseManager.getLease(lease.getLeaseKey()); + Lease fromDynamo = leaseRefresher.getLease(lease.leaseKey()); - Assert.assertEquals(lease, fromDynamo); + assertEquals(lease, fromDynamo); } /** - * Tests leaseManager.holdLease when the lease has changed out from under us. + * Tests leaseRefresher.holdLease when the lease has changed out from under us. */ @Test public void testHoldUpdatedLease() throws LeasingException { - TestHarnessBuilder builder = new TestHarnessBuilder(leaseManager); - KinesisClientLease lease = builder.withLease("1").build().get("1"); + TestHarnessBuilder builder = new TestHarnessBuilder(leaseRefresher); + Lease lease = builder.withLease("1").build().get("1"); - KinesisClientLease leaseCopy = leaseManager.getLease(lease.getLeaseKey()); + Lease leaseCopy = leaseRefresher.getLease(lease.leaseKey()); // lose lease - leaseManager.takeLease(lease, "bar"); + leaseRefresher.takeLease(lease, "bar"); - Assert.assertFalse(leaseManager.renewLease(leaseCopy)); + assertFalse(leaseRefresher.renewLease(leaseCopy)); } /** @@ -131,19 +141,19 @@ public class DynamoDBLeaseManagerIntegrationTest extends LeaseIntegrationTest { } private void testTakeLease(boolean owned) throws LeasingException { - TestHarnessBuilder builder = new TestHarnessBuilder(leaseManager); - KinesisClientLease lease = builder.withLease("1", owned ? "originalOwner" : null).build().get("1"); - Long originalLeaseCounter = lease.getLeaseCounter(); + TestHarnessBuilder builder = new TestHarnessBuilder(leaseRefresher); + Lease lease = builder.withLease("1", owned ? "originalOwner" : null).build().get("1"); + Long originalLeaseCounter = lease.leaseCounter(); String newOwner = "newOwner"; - leaseManager.takeLease(lease, newOwner); - Assert.assertTrue(originalLeaseCounter + 1 == lease.getLeaseCounter()); - Assert.assertTrue((owned ? 1 : 0) == lease.getOwnerSwitchesSinceCheckpoint()); - Assert.assertEquals(newOwner, lease.getLeaseOwner()); + leaseRefresher.takeLease(lease, newOwner); + assertTrue(originalLeaseCounter + 1 == lease.leaseCounter()); + assertTrue((owned ? 1 : 0) == lease.ownerSwitchesSinceCheckpoint()); + assertEquals(newOwner, lease.leaseOwner()); - Lease fromDynamo = leaseManager.getLease(lease.getLeaseKey()); + Lease fromDynamo = leaseRefresher.getLease(lease.leaseKey()); - Assert.assertEquals(lease, fromDynamo); + assertEquals(lease, fromDynamo); } /** @@ -151,25 +161,25 @@ public class DynamoDBLeaseManagerIntegrationTest extends LeaseIntegrationTest { */ @Test public void testTakeUpdatedLease() throws LeasingException { - TestHarnessBuilder builder = new TestHarnessBuilder(leaseManager); - KinesisClientLease lease = builder.withLease("1").build().get("1"); + TestHarnessBuilder builder = new TestHarnessBuilder(leaseRefresher); + Lease lease = builder.withLease("1").build().get("1"); - KinesisClientLease leaseCopy = leaseManager.getLease(lease.getLeaseKey()); + Lease leaseCopy = leaseRefresher.getLease(lease.leaseKey()); String newOwner = "newOwner"; - leaseManager.takeLease(lease, newOwner); + leaseRefresher.takeLease(lease, newOwner); - Assert.assertFalse(leaseManager.takeLease(leaseCopy, newOwner)); + assertFalse(leaseRefresher.takeLease(leaseCopy, newOwner)); } /** * Tests evictLease when the lease is currently unowned. */ public void testEvictUnownedLease() throws LeasingException { - TestHarnessBuilder builder = new TestHarnessBuilder(leaseManager); - KinesisClientLease lease = builder.withLease("1", null).build().get("1"); + TestHarnessBuilder builder = new TestHarnessBuilder(leaseRefresher); + Lease lease = builder.withLease("1", null).build().get("1"); - Assert.assertFalse(leaseManager.evictLease(lease)); + assertFalse(leaseRefresher.evictLease(lease)); } /** @@ -177,17 +187,17 @@ public class DynamoDBLeaseManagerIntegrationTest extends LeaseIntegrationTest { */ @Test public void testEvictOwnedLease() throws LeasingException { - TestHarnessBuilder builder = new TestHarnessBuilder(leaseManager); - KinesisClientLease lease = builder.withLease("1").build().get("1"); - Long originalLeaseCounter = lease.getLeaseCounter(); + TestHarnessBuilder builder = new TestHarnessBuilder(leaseRefresher); + Lease lease = builder.withLease("1").build().get("1"); + Long originalLeaseCounter = lease.leaseCounter(); - leaseManager.evictLease(lease); - Assert.assertNull(lease.getLeaseOwner()); - Assert.assertTrue(originalLeaseCounter + 1 == lease.getLeaseCounter()); + leaseRefresher.evictLease(lease); + assertNull(lease.leaseOwner()); + assertTrue(originalLeaseCounter + 1 == lease.leaseCounter()); - Lease fromDynamo = leaseManager.getLease(lease.getLeaseKey()); + Lease fromDynamo = leaseRefresher.getLease(lease.leaseKey()); - Assert.assertEquals(lease, fromDynamo); + assertEquals(lease, fromDynamo); } /** @@ -197,12 +207,12 @@ public class DynamoDBLeaseManagerIntegrationTest extends LeaseIntegrationTest { */ @Test public void testEvictChangedLease() throws LeasingException { - TestHarnessBuilder builder = new TestHarnessBuilder(leaseManager); - KinesisClientLease lease = builder.withLease("1").build().get("1"); + TestHarnessBuilder builder = new TestHarnessBuilder(leaseRefresher); + Lease lease = builder.withLease("1").build().get("1"); // Change the owner only - this should cause our optimistic lock to fail. - lease.setLeaseOwner("otherOwner"); - Assert.assertFalse(leaseManager.evictLease(lease)); + lease.leaseOwner("otherOwner"); + assertFalse(leaseRefresher.evictLease(lease)); } /** @@ -210,13 +220,13 @@ public class DynamoDBLeaseManagerIntegrationTest extends LeaseIntegrationTest { */ @Test public void testDeleteLease() throws LeasingException { - TestHarnessBuilder builder = new TestHarnessBuilder(leaseManager); - KinesisClientLease lease = builder.withLease("1").build().get("1"); + TestHarnessBuilder builder = new TestHarnessBuilder(leaseRefresher); + Lease lease = builder.withLease("1").build().get("1"); - leaseManager.deleteLease(lease); + leaseRefresher.deleteLease(lease); - KinesisClientLease newLease = leaseManager.getLease(lease.getLeaseKey()); - Assert.assertNull(newLease); + Lease newLease = leaseRefresher.getLease(lease.leaseKey()); + assertNull(newLease); } /** @@ -224,26 +234,25 @@ public class DynamoDBLeaseManagerIntegrationTest extends LeaseIntegrationTest { */ @Test public void testDeleteNonexistentLease() throws LeasingException { - KinesisClientLease lease = new KinesisClientLease(); - lease.setLeaseKey("1"); + Lease lease = new Lease(); + lease.leaseKey("1"); // The lease has not been written to DDB - try to delete it and expect success. - leaseManager.deleteLease(lease); + leaseRefresher.deleteLease(lease); } @Test public void testWaitUntilLeaseTableExists() throws LeasingException { - KinesisClientDynamoDBLeaseManager manager = new KinesisClientDynamoDBLeaseManager("nagl_ShardProgress", ddbClient, true) { - + DynamoDBLeaseRefresher refresher = new DynamoDBLeaseRefresher("nagl_ShardProgress", ddbClient, new DynamoDBLeaseSerializer(), true) { @Override long sleep(long timeToSleepMillis) { - Assert.fail("Should not sleep"); + fail("Should not sleep"); return 0L; } }; - Assert.assertTrue(manager.waitUntilLeaseTableExists(1, 1)); + assertTrue(refresher.waitUntilLeaseTableExists(1, 1)); } @Test @@ -252,18 +261,17 @@ public class DynamoDBLeaseManagerIntegrationTest extends LeaseIntegrationTest { * Just using AtomicInteger for the indirection it provides. */ final AtomicInteger sleepCounter = new AtomicInteger(0); - KinesisClientDynamoDBLeaseManager manager = new KinesisClientDynamoDBLeaseManager("nonexistentTable", ddbClient, true) { - + DynamoDBLeaseRefresher refresher = new DynamoDBLeaseRefresher("nonexistentTable", ddbClient, new DynamoDBLeaseSerializer(), true) { @Override long sleep(long timeToSleepMillis) { - Assert.assertEquals(1000L, timeToSleepMillis); + assertEquals(1000L, timeToSleepMillis); sleepCounter.incrementAndGet(); return 1000L; } }; - Assert.assertFalse(manager.waitUntilLeaseTableExists(2, 1)); - Assert.assertEquals(1, sleepCounter.get()); + assertFalse(refresher.waitUntilLeaseTableExists(2, 1)); + assertEquals(1, sleepCounter.get()); } } diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/DynamoDBLeaseRenewerIntegrationTest.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseRenewerIntegrationTest.java similarity index 69% rename from amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/DynamoDBLeaseRenewerIntegrationTest.java rename to amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseRenewerIntegrationTest.java index d5c76e0d..80ac5edb 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/DynamoDBLeaseRenewerIntegrationTest.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseRenewerIntegrationTest.java @@ -12,34 +12,37 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package software.amazon.kinesis.leases; - -import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; -import software.amazon.kinesis.leases.exceptions.LeasingException; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +package software.amazon.kinesis.leases.dynamodb; import java.util.Collections; import java.util.Map; import java.util.concurrent.Executors; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import software.amazon.kinesis.leases.Lease; +import software.amazon.kinesis.leases.LeaseIntegrationTest; +import software.amazon.kinesis.leases.LeaseRenewer; +import software.amazon.kinesis.leases.exceptions.LeasingException; +import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; + public class DynamoDBLeaseRenewerIntegrationTest extends LeaseIntegrationTest { // This test case's leases last 2 seconds private static final long LEASE_DURATION_MILLIS = 2000L; - private LeaseRenewer renewer; + private LeaseRenewer renewer; @Before public void setUp() { - renewer = new DynamoDBLeaseRenewer( - leaseManager, "foo", LEASE_DURATION_MILLIS, Executors.newCachedThreadPool()); + renewer = new DynamoDBLeaseRenewer(leaseRefresher, "foo", LEASE_DURATION_MILLIS, Executors.newCachedThreadPool()); } @Test public void testSimpleRenew() throws LeasingException { - TestHarnessBuilder builder = new TestHarnessBuilder(leaseManager); + TestHarnessBuilder builder = new TestHarnessBuilder(leaseRefresher); builder.withLease("1", "foo").build(); @@ -49,22 +52,22 @@ public class DynamoDBLeaseRenewerIntegrationTest extends LeaseIntegrationTest { @Test public void testLeaseLoss() throws LeasingException { - TestHarnessBuilder builder = new TestHarnessBuilder(leaseManager); + TestHarnessBuilder builder = new TestHarnessBuilder(leaseRefresher); builder.withLease("1", "foo").withLease("2", "foo").build(); builder.addLeasesToRenew(renewer, "1", "2"); - KinesisClientLease renewedLease = builder.renewMutateAssert(renewer, "1", "2").get("2"); + Lease renewedLease = builder.renewMutateAssert(renewer, "1", "2").get("2"); // lose lease 2 - leaseManager.takeLease(renewedLease, "bar"); + leaseRefresher.takeLease(renewedLease, "bar"); builder.renewMutateAssert(renewer, "1"); } @Test public void testClear() throws LeasingException { - TestHarnessBuilder builder = new TestHarnessBuilder(leaseManager); + TestHarnessBuilder builder = new TestHarnessBuilder(leaseRefresher); builder.withLease("1", "foo").build(); builder.addLeasesToRenew(renewer, "1"); @@ -76,143 +79,143 @@ public class DynamoDBLeaseRenewerIntegrationTest extends LeaseIntegrationTest { @Test public void testGetCurrentlyHeldLease() throws LeasingException { - TestHarnessBuilder builder = new TestHarnessBuilder(leaseManager); + TestHarnessBuilder builder = new TestHarnessBuilder(leaseRefresher); builder.withLease("1", "foo").build(); builder.addLeasesToRenew(renewer, "1"); builder.renewMutateAssert(renewer, "1"); // this should be a copy that doesn't get updated - KinesisClientLease lease = renewer.getCurrentlyHeldLease("1"); - Assert.assertEquals((Long) 1L, lease.getLeaseCounter()); + Lease lease = renewer.getCurrentlyHeldLease("1"); + Assert.assertEquals((Long) 1L, lease.leaseCounter()); // do one renewal and make sure the old copy doesn't get updated builder.renewMutateAssert(renewer, "1"); - Assert.assertEquals((Long) 1L, lease.getLeaseCounter()); + Assert.assertEquals((Long) 1L, lease.leaseCounter()); } @Test public void testGetCurrentlyHeldLeases() throws LeasingException { - TestHarnessBuilder builder = new TestHarnessBuilder(leaseManager); + TestHarnessBuilder builder = new TestHarnessBuilder(leaseRefresher); builder.withLease("1", "foo").withLease("2", "foo").build(); builder.addLeasesToRenew(renewer, "1", "2"); - KinesisClientLease lease2 = builder.renewMutateAssert(renewer, "1", "2").get("2"); + Lease lease2 = builder.renewMutateAssert(renewer, "1", "2").get("2"); // This should be a copy that doesn't get updated - Map heldLeases = renewer.getCurrentlyHeldLeases(); + Map heldLeases = renewer.getCurrentlyHeldLeases(); Assert.assertEquals(2, heldLeases.size()); - Assert.assertEquals((Long) 1L, heldLeases.get("1").getLeaseCounter()); - Assert.assertEquals((Long) 1L, heldLeases.get("2").getLeaseCounter()); + Assert.assertEquals((Long) 1L, heldLeases.get("1").leaseCounter()); + Assert.assertEquals((Long) 1L, heldLeases.get("2").leaseCounter()); // lose lease 2 - leaseManager.takeLease(lease2, "bar"); + leaseRefresher.takeLease(lease2, "bar"); // Do another renewal and make sure the copy doesn't change builder.renewMutateAssert(renewer, "1"); Assert.assertEquals(2, heldLeases.size()); - Assert.assertEquals((Long) 1L, heldLeases.get("1").getLeaseCounter()); - Assert.assertEquals((Long) 1L, heldLeases.get("2").getLeaseCounter()); + Assert.assertEquals((Long) 1L, heldLeases.get("1").leaseCounter()); + Assert.assertEquals((Long) 1L, heldLeases.get("2").leaseCounter()); } @Test public void testUpdateLease() throws LeasingException { - TestHarnessBuilder builder = new TestHarnessBuilder(leaseManager); + TestHarnessBuilder builder = new TestHarnessBuilder(leaseRefresher); builder.withLease("1", "foo").build(); builder.addLeasesToRenew(renewer, "1"); builder.renewMutateAssert(renewer, "1"); - KinesisClientLease expected = renewer.getCurrentlyHeldLease("1"); - expected.setCheckpoint(new ExtendedSequenceNumber("new checkpoint")); - Assert.assertTrue(renewer.updateLease(expected, expected.getConcurrencyToken())); + Lease expected = renewer.getCurrentlyHeldLease("1"); + expected.checkpoint(new ExtendedSequenceNumber("new checkpoint")); + Assert.assertTrue(renewer.updateLease(expected, expected.concurrencyToken())); // Assert that the counter and data have changed immediately after the update... - KinesisClientLease actual = renewer.getCurrentlyHeldLease("1"); - expected.setLeaseCounter(expected.getLeaseCounter() + 1); + Lease actual = renewer.getCurrentlyHeldLease("1"); + expected.leaseCounter(expected.leaseCounter() + 1); Assert.assertEquals(expected, actual); // ...and after another round of renewal renewer.renewLeases(); actual = renewer.getCurrentlyHeldLease("1"); - expected.setLeaseCounter(expected.getLeaseCounter() + 1); + expected.leaseCounter(expected.leaseCounter() + 1); Assert.assertEquals(expected, actual); } @Test public void testUpdateLostLease() throws LeasingException { - TestHarnessBuilder builder = new TestHarnessBuilder(leaseManager); + TestHarnessBuilder builder = new TestHarnessBuilder(leaseRefresher); builder.withLease("1", "foo").build(); builder.addLeasesToRenew(renewer, "1"); builder.renewMutateAssert(renewer, "1"); - KinesisClientLease lease = renewer.getCurrentlyHeldLease("1"); + Lease lease = renewer.getCurrentlyHeldLease("1"); // cause lease loss such that the renewer doesn't realize he's lost the lease when update is called - leaseManager.renewLease(lease); + leaseRefresher.renewLease(lease); // renewer still thinks he has the lease Assert.assertNotNull(renewer.getCurrentlyHeldLease("1")); - lease.setCheckpoint(new ExtendedSequenceNumber("new checkpoint")); + lease.checkpoint(new ExtendedSequenceNumber("new checkpoint")); // update fails - Assert.assertFalse(renewer.updateLease(lease, lease.getConcurrencyToken())); + Assert.assertFalse(renewer.updateLease(lease, lease.concurrencyToken())); // renewer no longer thinks he has the lease Assert.assertNull(renewer.getCurrentlyHeldLease("1")); } @Test public void testUpdateOldLease() throws LeasingException { - TestHarnessBuilder builder = new TestHarnessBuilder(leaseManager); + TestHarnessBuilder builder = new TestHarnessBuilder(leaseRefresher); builder.withLease("1", "foo").build(); builder.addLeasesToRenew(renewer, "1"); builder.renewMutateAssert(renewer, "1"); - KinesisClientLease lease = renewer.getCurrentlyHeldLease("1"); + Lease lease = renewer.getCurrentlyHeldLease("1"); // cause lease loss such that the renewer knows the lease has been lost when update is called - leaseManager.takeLease(lease, "bar"); + leaseRefresher.takeLease(lease, "bar"); builder.renewMutateAssert(renewer); - lease.setCheckpoint(new ExtendedSequenceNumber("new checkpoint")); - Assert.assertFalse(renewer.updateLease(lease, lease.getConcurrencyToken())); + lease.checkpoint(new ExtendedSequenceNumber("new checkpoint")); + Assert.assertFalse(renewer.updateLease(lease, lease.concurrencyToken())); } @Test public void testUpdateRegainedLease() throws LeasingException { - TestHarnessBuilder builder = new TestHarnessBuilder(leaseManager); + TestHarnessBuilder builder = new TestHarnessBuilder(leaseRefresher); builder.withLease("1", "foo").build(); builder.addLeasesToRenew(renewer, "1"); builder.renewMutateAssert(renewer, "1"); - KinesisClientLease lease = renewer.getCurrentlyHeldLease("1"); + Lease lease = renewer.getCurrentlyHeldLease("1"); // cause lease loss such that the renewer knows the lease has been lost when update is called - leaseManager.takeLease(lease, "bar"); + leaseRefresher.takeLease(lease, "bar"); builder.renewMutateAssert(renewer); // regain the lease builder.addLeasesToRenew(renewer, "1"); - lease.setCheckpoint(new ExtendedSequenceNumber("new checkpoint")); - Assert.assertFalse(renewer.updateLease(lease, lease.getConcurrencyToken())); + lease.checkpoint(new ExtendedSequenceNumber("new checkpoint")); + Assert.assertFalse(renewer.updateLease(lease, lease.concurrencyToken())); } @Test public void testIgnoreNoRenewalTimestamp() throws LeasingException { - TestHarnessBuilder builder = new TestHarnessBuilder(leaseManager); + TestHarnessBuilder builder = new TestHarnessBuilder(leaseRefresher); - KinesisClientLease lease = builder.withLease("1", "foo").build().get("1"); - lease.setLastCounterIncrementNanos(null); + Lease lease = builder.withLease("1", "foo").build().get("1"); + lease.lastCounterIncrementNanos(null); renewer.addLeasesToRenew(Collections.singleton(lease)); @@ -221,7 +224,7 @@ public class DynamoDBLeaseRenewerIntegrationTest extends LeaseIntegrationTest { @Test public void testLeaseTimeout() throws LeasingException, InterruptedException { - TestHarnessBuilder builder = new TestHarnessBuilder(leaseManager); + TestHarnessBuilder builder = new TestHarnessBuilder(leaseRefresher); builder.withLease("1", "foo").build(); @@ -239,13 +242,13 @@ public class DynamoDBLeaseRenewerIntegrationTest extends LeaseIntegrationTest { final String shardId = "shd-0-0"; final String owner = "foo:8000"; - TestHarnessBuilder builder = new TestHarnessBuilder(leaseManager); + TestHarnessBuilder builder = new TestHarnessBuilder(leaseRefresher); builder.withLease(shardId, owner); - Map leases = builder.build(); - DynamoDBLeaseRenewer renewer =new DynamoDBLeaseRenewer( - leaseManager, owner, 30000L, Executors.newCachedThreadPool()); + Map leases = builder.build(); + DynamoDBLeaseRenewer renewer = new DynamoDBLeaseRenewer( + leaseRefresher, owner, 30000L, Executors.newCachedThreadPool()); renewer.initialize(); - Map heldLeases = renewer.getCurrentlyHeldLeases(); + Map heldLeases = renewer.getCurrentlyHeldLeases(); Assert.assertEquals(leases.size(), heldLeases.size()); Assert.assertEquals(leases.keySet(), heldLeases.keySet()); } diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/DynamoDBLeaseRenewerTest.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseRenewerTest.java similarity index 55% rename from amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/DynamoDBLeaseRenewerTest.java rename to amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseRenewerTest.java index cb75ab4b..a2d3e0d4 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/DynamoDBLeaseRenewerTest.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseRenewerTest.java @@ -12,61 +12,56 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package software.amazon.kinesis.leases; +package software.amazon.kinesis.leases.dynamodb; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.UUID; -import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import org.junit.After; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.mockito.Mockito; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; +import software.amazon.kinesis.leases.Lease; +import software.amazon.kinesis.leases.LeaseRefresher; +import software.amazon.kinesis.leases.dynamodb.DynamoDBLeaseRenewer; import software.amazon.kinesis.leases.exceptions.DependencyException; import software.amazon.kinesis.leases.exceptions.InvalidStateException; import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException; +@RunWith(MockitoJUnitRunner.class) public class DynamoDBLeaseRenewerTest { + private final String workerIdentifier = "WorkerId"; + private final long leaseDurationMillis = 10000; + private DynamoDBLeaseRenewer renewer; + private List leasesToRenew; - LeaseManager leaseManager; - String workerIdentifier; - long leaseDurationMillis; - ExecutorService leaseRenewalExecService; - DynamoDBLeaseRenewer renewer; - List leasesToRenew; - - private static Lease newLease(String leaseKey, - String leaseOwner, - Long leaseCounter, - UUID concurrencyToken, - Long lastCounterIncrementNanos) { - Lease lease = new Lease(); - lease.setLeaseKey(leaseKey); - lease.setLeaseOwner(leaseOwner); - lease.setLeaseCounter(leaseCounter); - lease.setConcurrencyToken(concurrencyToken); - lease.setLastCounterIncrementNanos(lastCounterIncrementNanos); - return lease; - } + @Mock + private LeaseRefresher leaseRefresher; private static Lease newLease(String leaseKey) { - return newLease(leaseKey, "leaseOwner", 0L, UUID.randomUUID(), System.nanoTime()); + return new Lease(leaseKey, "LeaseOwner", 0L, UUID.randomUUID(), System.nanoTime(), null, null, null, new HashSet<>()); } - @SuppressWarnings("unchecked") @Before public void before() { - leaseManager = Mockito.mock(LeaseManager.class); - workerIdentifier = "workerId"; - leaseDurationMillis = 10000; - leaseRenewalExecService = Executors.newSingleThreadExecutor(); leasesToRenew = null; - renewer = new DynamoDBLeaseRenewer<>(leaseManager, + renewer = new DynamoDBLeaseRenewer(leaseRefresher, workerIdentifier, leaseDurationMillis, Executors.newCachedThreadPool()); @@ -77,8 +72,8 @@ public class DynamoDBLeaseRenewerTest { if (leasesToRenew == null) { return; } - for (Lease l : leasesToRenew) { - Mockito.verify(leaseManager, Mockito.times(1)).renewLease(l); + for (Lease lease : leasesToRenew) { + verify(leaseRefresher, times(1)).renewLease(eq(lease)); } } @@ -91,16 +86,15 @@ public class DynamoDBLeaseRenewerTest { */ Lease lease1 = newLease("1"); Lease lease2 = newLease("2"); - leasesToRenew = - Arrays.asList(lease1,lease2); + leasesToRenew = Arrays.asList(lease1,lease2); renewer.addLeasesToRenew(leasesToRenew); - Mockito.doReturn(true).when(leaseManager).renewLease(lease1); - Mockito.doReturn(true).when(leaseManager).renewLease(lease2); + doReturn(true).when(leaseRefresher).renewLease(lease1); + doReturn(true).when(leaseRefresher).renewLease(lease2); renewer.renewLeases(); - Assert.assertEquals(2, renewer.getCurrentlyHeldLeases().size()); + assertEquals(2, renewer.getCurrentlyHeldLeases().size()); } @Test @@ -108,19 +102,19 @@ public class DynamoDBLeaseRenewerTest { String leaseKey = "expiredLease"; long initialCounterIncrementNanos = 5L; // "expired" time. Lease lease1 = newLease(leaseKey); - lease1.setLastCounterIncrementNanos(initialCounterIncrementNanos); + lease1.lastCounterIncrementNanos(initialCounterIncrementNanos); leasesToRenew = new ArrayList<>(); leasesToRenew.add(lease1); - Mockito.doReturn(true).when(leaseManager).renewLease(lease1); + doReturn(true).when(leaseRefresher).renewLease(lease1); renewer.addLeasesToRenew(leasesToRenew); - Assert.assertTrue(lease1.isExpired(1, System.nanoTime())); - Assert.assertNull(renewer.getCurrentlyHeldLease(leaseKey)); + assertTrue(lease1.isExpired(1, System.nanoTime())); + assertNull(renewer.getCurrentlyHeldLease(leaseKey)); renewer.renewLeases(); // Don't renew lease(s) with same key if getCurrentlyHeldLease returned null previously - Assert.assertNull(renewer.getCurrentlyHeldLease(leaseKey)); - Assert.assertFalse(renewer.getCurrentlyHeldLeases().containsKey(leaseKey)); + assertNull(renewer.getCurrentlyHeldLease(leaseKey)); + assertFalse(renewer.getCurrentlyHeldLeases().containsKey(leaseKey)); // Clear the list to avoid triggering expectation mismatch in after(). leasesToRenew.clear(); diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/DynamoDBLeaseTakerIntegrationTest.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseTakerIntegrationTest.java similarity index 92% rename from amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/DynamoDBLeaseTakerIntegrationTest.java rename to amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseTakerIntegrationTest.java index de7319fc..496ae997 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/DynamoDBLeaseTakerIntegrationTest.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseTakerIntegrationTest.java @@ -12,7 +12,7 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package software.amazon.kinesis.leases; +package software.amazon.kinesis.leases.dynamodb; import java.util.Map; @@ -20,21 +20,23 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import software.amazon.kinesis.leases.Lease; +import software.amazon.kinesis.leases.LeaseIntegrationTest; import software.amazon.kinesis.leases.exceptions.LeasingException; public class DynamoDBLeaseTakerIntegrationTest extends LeaseIntegrationTest { private static final long LEASE_DURATION_MILLIS = 1000L; - private DynamoDBLeaseTaker taker; + private DynamoDBLeaseTaker taker; @Before public void setUp() { - taker = new DynamoDBLeaseTaker(leaseManager, "foo", LEASE_DURATION_MILLIS); + taker = new DynamoDBLeaseTaker(leaseRefresher, "foo", LEASE_DURATION_MILLIS); } @Test public void testSimpleLeaseTake() throws LeasingException { - TestHarnessBuilder builder = new TestHarnessBuilder(leaseManager); + TestHarnessBuilder builder = new TestHarnessBuilder(leaseRefresher); builder.withLease("1", null).build(); @@ -43,7 +45,7 @@ public class DynamoDBLeaseTakerIntegrationTest extends LeaseIntegrationTest { @Test public void testNotTakeUpdatedLease() throws LeasingException { - TestHarnessBuilder builder = new TestHarnessBuilder(leaseManager); + TestHarnessBuilder builder = new TestHarnessBuilder(leaseRefresher); builder.withLease("1", "bar").build(); @@ -56,7 +58,7 @@ public class DynamoDBLeaseTakerIntegrationTest extends LeaseIntegrationTest { @Test public void testTakeOwnLease() throws LeasingException { - TestHarnessBuilder builder = new TestHarnessBuilder(leaseManager); + TestHarnessBuilder builder = new TestHarnessBuilder(leaseRefresher); builder.withLease("1", taker.getWorkerIdentifier()).build(); @@ -67,7 +69,7 @@ public class DynamoDBLeaseTakerIntegrationTest extends LeaseIntegrationTest { @Test public void testNotTakeNewOwnedLease() throws LeasingException, InterruptedException { - TestHarnessBuilder builder = new TestHarnessBuilder(leaseManager); + TestHarnessBuilder builder = new TestHarnessBuilder(leaseRefresher); builder.withLease("1", "bar").build(); @@ -85,7 +87,7 @@ public class DynamoDBLeaseTakerIntegrationTest extends LeaseIntegrationTest { */ @Test public void testNonGreedyTake() throws LeasingException { - TestHarnessBuilder builder = new TestHarnessBuilder(leaseManager); + TestHarnessBuilder builder = new TestHarnessBuilder(leaseRefresher); for (int i = 0; i < 3; i++) { builder.withLease(Integer.toString(i), null); @@ -103,7 +105,7 @@ public class DynamoDBLeaseTakerIntegrationTest extends LeaseIntegrationTest { */ @Test public void testNoStealWhenOffByOne() throws LeasingException { - TestHarnessBuilder builder = new TestHarnessBuilder(leaseManager); + TestHarnessBuilder builder = new TestHarnessBuilder(leaseRefresher); builder.withLease("1", "bar") .withLease("2", "bar") @@ -124,7 +126,7 @@ public class DynamoDBLeaseTakerIntegrationTest extends LeaseIntegrationTest { */ @Test public void testSteal() throws LeasingException { - TestHarnessBuilder builder = new TestHarnessBuilder(leaseManager); + TestHarnessBuilder builder = new TestHarnessBuilder(leaseRefresher); builder.withLease("1", "bar"); for (int i = 2; i <= 6; i++) { @@ -135,7 +137,7 @@ public class DynamoDBLeaseTakerIntegrationTest extends LeaseIntegrationTest { builder.build(); // Assert that one lease was stolen from baz. - Map takenLeases = builder.takeMutateAssert(taker, 1); + Map takenLeases = builder.takeMutateAssert(taker, 1); // Assert that it was one of baz's leases (shardId != 1) String shardIdStolen = takenLeases.keySet().iterator().next(); @@ -148,7 +150,7 @@ public class DynamoDBLeaseTakerIntegrationTest extends LeaseIntegrationTest { */ @Test public void testNoStealWhenExpiredLeases() throws LeasingException { - TestHarnessBuilder builder = new TestHarnessBuilder(leaseManager); + TestHarnessBuilder builder = new TestHarnessBuilder(leaseRefresher); builder.withLease("1", null); for (int i = 2; i <= 4; i++) { diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/DynamoDBLeaseTakerTest.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseTakerTest.java similarity index 94% rename from amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/DynamoDBLeaseTakerTest.java rename to amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseTakerTest.java index a4b91c6e..458d9cdf 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/DynamoDBLeaseTakerTest.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseTakerTest.java @@ -12,7 +12,7 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package software.amazon.kinesis.leases; +package software.amazon.kinesis.leases.dynamodb; import java.util.ArrayList; import java.util.List; @@ -24,6 +24,7 @@ import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import software.amazon.kinesis.leases.dynamodb.DynamoDBLeaseTaker; /** * diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/dynamodb/TestHarnessBuilder.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/dynamodb/TestHarnessBuilder.java new file mode 100644 index 00000000..1ea73a3e --- /dev/null +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/leases/dynamodb/TestHarnessBuilder.java @@ -0,0 +1,179 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package software.amazon.kinesis.leases.dynamodb; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; + +import software.amazon.kinesis.leases.Lease; +import software.amazon.kinesis.leases.LeaseRenewer; +import software.amazon.kinesis.leases.exceptions.DependencyException; +import software.amazon.kinesis.leases.exceptions.InvalidStateException; +import software.amazon.kinesis.leases.exceptions.LeasingException; +import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber; + +public class TestHarnessBuilder { + + private long currentTimeNanos; + + private Map leases = new HashMap<>(); + private DynamoDBLeaseRefresher leaseRefresher; + private Map originalLeases = new HashMap<>(); + + private Callable timeProvider = new Callable() { + + @Override + public Long call() throws Exception { + return currentTimeNanos; + } + + }; + + public TestHarnessBuilder(final DynamoDBLeaseRefresher leaseRefresher) { + this.leaseRefresher = leaseRefresher; + } + + public TestHarnessBuilder withLease(String shardId) { + return withLease(shardId, "leaseOwner"); + } + + public TestHarnessBuilder withLease(String shardId, String owner) { + Lease lease = createLease(shardId, owner); + Lease originalLease = createLease(shardId, owner); + + leases.put(shardId, lease); + originalLeases.put(shardId, originalLease); + return this; + } + + private Lease createLease(String shardId, String owner) { + Lease lease = new Lease(); + lease.checkpoint(new ExtendedSequenceNumber("checkpoint")); + lease.ownerSwitchesSinceCheckpoint(0L); + lease.leaseCounter(0L); + lease.leaseOwner(owner); + lease.parentShardIds(Collections.singleton("parentShardId")); + lease.leaseKey(shardId); + + return lease; + } + + public Map build() throws LeasingException { + for (Lease lease : leases.values()) { + leaseRefresher.createLeaseIfNotExists(lease); + if (lease.leaseOwner() != null) { + lease.lastCounterIncrementNanos(System.nanoTime()); + } + } + + currentTimeNanos = System.nanoTime(); + + return leases; + } + + public void passTime(long millis) { + currentTimeNanos += millis * 1000000; + } + + public Map takeMutateAssert(DynamoDBLeaseTaker taker, int numToTake) + throws LeasingException { + Map result = taker.takeLeases(timeProvider); + assertEquals(numToTake, result.size()); + + for (Lease actual : result.values()) { + Lease original = leases.get(actual.leaseKey()); + assertNotNull(original); + + mutateAssert(taker.getWorkerIdentifier(), original, actual); + } + + return result; + } + + public Map takeMutateAssert(DynamoDBLeaseTaker taker, String... takenShardIds) + throws LeasingException { + Map result = taker.takeLeases(timeProvider); + assertEquals(takenShardIds.length, result.size()); + + for (String shardId : takenShardIds) { + Lease original = leases.get(shardId); + assertNotNull(original); + + Lease actual = result.get(shardId); + assertNotNull(actual); + + mutateAssert(taker.getWorkerIdentifier(), original, actual); + } + + return result; + } + + private void mutateAssert(String newWorkerIdentifier, Lease original, Lease actual) { + original.leaseCounter(original.leaseCounter() + 1); + if (original.leaseOwner() != null && !newWorkerIdentifier.equals(original.leaseOwner())) { + original.ownerSwitchesSinceCheckpoint(original.ownerSwitchesSinceCheckpoint() + 1); + } + original.leaseOwner(newWorkerIdentifier); + + assertEquals(original, actual); // Assert the contents of the lease + } + + public void addLeasesToRenew(LeaseRenewer renewer, String... shardIds) + throws DependencyException, InvalidStateException { + List leasesToRenew = new ArrayList(); + + for (String shardId : shardIds) { + Lease lease = leases.get(shardId); + assertNotNull(lease); + leasesToRenew.add(lease); + } + + renewer.addLeasesToRenew(leasesToRenew); + } + + public Map renewMutateAssert(LeaseRenewer renewer, String... renewedShardIds) + throws DependencyException, InvalidStateException { + renewer.renewLeases(); + + Map heldLeases = renewer.getCurrentlyHeldLeases(); + assertEquals(renewedShardIds.length, heldLeases.size()); + + for (String shardId : renewedShardIds) { + Lease original = originalLeases.get(shardId); + assertNotNull(original); + + Lease actual = heldLeases.get(shardId); + assertNotNull(actual); + + original.leaseCounter(original.leaseCounter() + 1); + assertEquals(original, actual); + } + + return heldLeases; + } + + public void renewAllLeases() throws LeasingException { + for (Lease lease : leases.values()) { + leaseRefresher.renewLease(lease); + } + } +} diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/BlockOnParentShardTaskTest.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/BlockOnParentShardTaskTest.java index c64d9940..dcf94ff3 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/BlockOnParentShardTaskTest.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/BlockOnParentShardTaskTest.java @@ -26,8 +26,8 @@ import java.util.List; import org.junit.Before; import org.junit.Test; -import software.amazon.kinesis.leases.LeaseManager; -import software.amazon.kinesis.leases.KinesisClientLease; +import software.amazon.kinesis.leases.Lease; +import software.amazon.kinesis.leases.LeaseRefresher; import software.amazon.kinesis.leases.ShardInfo; import software.amazon.kinesis.leases.exceptions.DependencyException; import software.amazon.kinesis.leases.exceptions.InvalidStateException; @@ -58,10 +58,10 @@ public class BlockOnParentShardTaskTest { @Test public final void testCallNoParents() throws DependencyException, InvalidStateException, ProvisionedThroughputException { - LeaseManager leaseManager = mock(LeaseManager.class); - when(leaseManager.getLease(shardId)).thenReturn(null); + LeaseRefresher leaseRefresher = mock(LeaseRefresher.class); + when(leaseRefresher.getLease(shardId)).thenReturn(null); - BlockOnParentShardTask task = new BlockOnParentShardTask(shardInfo, leaseManager, backoffTimeInMillis); + BlockOnParentShardTask task = new BlockOnParentShardTask(shardInfo, leaseRefresher, backoffTimeInMillis); TaskResult result = task.call(); assertNull(result.getException()); } @@ -83,26 +83,26 @@ public class BlockOnParentShardTaskTest { List parentShardIds = new ArrayList<>(); TaskResult result = null; - KinesisClientLease parent1Lease = new KinesisClientLease(); - parent1Lease.setCheckpoint(ExtendedSequenceNumber.SHARD_END); - KinesisClientLease parent2Lease = new KinesisClientLease(); - parent2Lease.setCheckpoint(ExtendedSequenceNumber.SHARD_END); + Lease parent1Lease = new Lease(); + parent1Lease.checkpoint(ExtendedSequenceNumber.SHARD_END); + Lease parent2Lease = new Lease(); + parent2Lease.checkpoint(ExtendedSequenceNumber.SHARD_END); - LeaseManager leaseManager = mock(LeaseManager.class); - when(leaseManager.getLease(parent1ShardId)).thenReturn(parent1Lease); - when(leaseManager.getLease(parent2ShardId)).thenReturn(parent2Lease); + LeaseRefresher leaseRefresher = mock(LeaseRefresher.class); + when(leaseRefresher.getLease(parent1ShardId)).thenReturn(parent1Lease); + when(leaseRefresher.getLease(parent2ShardId)).thenReturn(parent2Lease); // test single parent parentShardIds.add(parent1ShardId); shardInfo = new ShardInfo(shardId, concurrencyToken, parentShardIds, ExtendedSequenceNumber.TRIM_HORIZON); - task = new BlockOnParentShardTask(shardInfo, leaseManager, backoffTimeInMillis); + task = new BlockOnParentShardTask(shardInfo, leaseRefresher, backoffTimeInMillis); result = task.call(); assertNull(result.getException()); // test two parents parentShardIds.add(parent2ShardId); shardInfo = new ShardInfo(shardId, concurrencyToken, parentShardIds, ExtendedSequenceNumber.TRIM_HORIZON); - task = new BlockOnParentShardTask(shardInfo, leaseManager, backoffTimeInMillis); + task = new BlockOnParentShardTask(shardInfo, leaseRefresher, backoffTimeInMillis); result = task.call(); assertNull(result.getException()); } @@ -124,27 +124,27 @@ public class BlockOnParentShardTaskTest { List parentShardIds = new ArrayList<>(); TaskResult result = null; - KinesisClientLease parent1Lease = new KinesisClientLease(); - parent1Lease.setCheckpoint(ExtendedSequenceNumber.LATEST); - KinesisClientLease parent2Lease = new KinesisClientLease(); + Lease parent1Lease = new Lease(); + parent1Lease.checkpoint(ExtendedSequenceNumber.LATEST); + Lease parent2Lease = new Lease(); // mock a sequence number checkpoint - parent2Lease.setCheckpoint(new ExtendedSequenceNumber("98182584034")); + parent2Lease.checkpoint(new ExtendedSequenceNumber("98182584034")); - LeaseManager leaseManager = mock(LeaseManager.class); - when(leaseManager.getLease(parent1ShardId)).thenReturn(parent1Lease); - when(leaseManager.getLease(parent2ShardId)).thenReturn(parent2Lease); + LeaseRefresher leaseRefresher = mock(LeaseRefresher.class); + when(leaseRefresher.getLease(parent1ShardId)).thenReturn(parent1Lease); + when(leaseRefresher.getLease(parent2ShardId)).thenReturn(parent2Lease); // test single parent parentShardIds.add(parent1ShardId); shardInfo = new ShardInfo(shardId, concurrencyToken, parentShardIds, ExtendedSequenceNumber.TRIM_HORIZON); - task = new BlockOnParentShardTask(shardInfo, leaseManager, backoffTimeInMillis); + task = new BlockOnParentShardTask(shardInfo, leaseRefresher, backoffTimeInMillis); result = task.call(); assertNotNull(result.getException()); // test two parents parentShardIds.add(parent2ShardId); shardInfo = new ShardInfo(shardId, concurrencyToken, parentShardIds, ExtendedSequenceNumber.TRIM_HORIZON); - task = new BlockOnParentShardTask(shardInfo, leaseManager, backoffTimeInMillis); + task = new BlockOnParentShardTask(shardInfo, leaseRefresher, backoffTimeInMillis); result = task.call(); assertNotNull(result.getException()); } @@ -165,19 +165,19 @@ public class BlockOnParentShardTaskTest { parentShardIds.add(parentShardId); ShardInfo shardInfo = new ShardInfo(shardId, concurrencyToken, parentShardIds, ExtendedSequenceNumber.TRIM_HORIZON); TaskResult result = null; - KinesisClientLease parentLease = new KinesisClientLease(); - LeaseManager leaseManager = mock(LeaseManager.class); - when(leaseManager.getLease(parentShardId)).thenReturn(parentLease); + Lease parentLease = new Lease(); + LeaseRefresher leaseRefresher = mock(LeaseRefresher.class); + when(leaseRefresher.getLease(parentShardId)).thenReturn(parentLease); // test when parent shard has not yet been fully processed - parentLease.setCheckpoint(new ExtendedSequenceNumber("98182584034")); - task = new BlockOnParentShardTask(shardInfo, leaseManager, backoffTimeInMillis); + parentLease.checkpoint(new ExtendedSequenceNumber("98182584034")); + task = new BlockOnParentShardTask(shardInfo, leaseRefresher, backoffTimeInMillis); result = task.call(); assertNotNull(result.getException()); // test when parent has been fully processed - parentLease.setCheckpoint(ExtendedSequenceNumber.SHARD_END); - task = new BlockOnParentShardTask(shardInfo, leaseManager, backoffTimeInMillis); + parentLease.checkpoint(ExtendedSequenceNumber.SHARD_END); + task = new BlockOnParentShardTask(shardInfo, leaseRefresher, backoffTimeInMillis); result = task.call(); assertNull(result.getException()); } diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/ConsumerStatesTest.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/ConsumerStatesTest.java index d2509299..8c291af3 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/ConsumerStatesTest.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/ConsumerStatesTest.java @@ -45,14 +45,13 @@ import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionIn import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStreamExtended; import software.amazon.kinesis.checkpoint.RecordProcessorCheckpointer; -import software.amazon.kinesis.leases.LeaseManager; -import software.amazon.kinesis.leases.KinesisClientLease; -import software.amazon.kinesis.leases.LeaseManagerProxy; +import software.amazon.kinesis.leases.LeaseRefresher; +import software.amazon.kinesis.leases.ShardDetector; import software.amazon.kinesis.leases.ShardInfo; import software.amazon.kinesis.metrics.IMetricsFactory; import software.amazon.kinesis.processor.Checkpointer; -import software.amazon.kinesis.processor.RecordProcessor; import software.amazon.kinesis.processor.IRecordProcessorCheckpointer; +import software.amazon.kinesis.processor.RecordProcessor; import software.amazon.kinesis.retrieval.GetRecordsCache; @RunWith(MockitoJUnitRunner.class) @@ -72,7 +71,7 @@ public class ConsumerStatesTest { @Mock private ShardInfo shardInfo; @Mock - private LeaseManager leaseManager; + private LeaseRefresher leaseRefresher; @Mock private Checkpointer checkpoint; @Mock @@ -84,7 +83,7 @@ public class ConsumerStatesTest { @Mock private AmazonKinesis amazonKinesis; @Mock - private LeaseManagerProxy leaseManagerProxy; + private ShardDetector shardDetector; @Mock private IMetricsFactory metricsFactory; @@ -102,18 +101,18 @@ public class ConsumerStatesTest { @Before public void setup() { - consumer = spy(new ShardConsumer(shardInfo, STREAM_NAME, leaseManager, executorService, getRecordsCache, + consumer = spy(new ShardConsumer(shardInfo, STREAM_NAME, leaseRefresher, executorService, getRecordsCache, recordProcessor, checkpoint, recordProcessorCheckpointer, parentShardPollIntervalMillis, taskBackoffTimeMillis, Optional.empty(), amazonKinesis, skipShardSyncAtWorkerInitializationIfLeasesExist, listShardsBackoffTimeInMillis, maxListShardsRetryAttempts, shouldCallProcessRecordsEvenForEmptyRecordList, idleTimeInMillis, INITIAL_POSITION_IN_STREAM, cleanupLeasesOfCompletedShards, ignoreUnexpectedChildShards, - leaseManagerProxy, metricsFactory)); + shardDetector, metricsFactory)); when(shardInfo.shardId()).thenReturn("shardId-000000000000"); } - private static final Class> LEASE_MANAGER_CLASS = (Class>) (Class) LeaseManager.class; + private static final Class LEASE_REFRESHER_CLASS = (Class) (Class) LeaseRefresher.class; @Test public void blockOnParentStateTest() { @@ -123,7 +122,7 @@ public class ConsumerStatesTest { assertThat(task, taskWith(BlockOnParentShardTask.class, ShardInfo.class, "shardInfo", equalTo(shardInfo))); assertThat(task, - taskWith(BlockOnParentShardTask.class, LEASE_MANAGER_CLASS, "leaseManager", equalTo(leaseManager))); + taskWith(BlockOnParentShardTask.class, LEASE_REFRESHER_CLASS, "leaseRefresher", equalTo(leaseRefresher))); assertThat(task, taskWith(BlockOnParentShardTask.class, Long.class, "parentShardPollIntervalMillis", equalTo(parentShardPollIntervalMillis))); @@ -300,7 +299,7 @@ public class ConsumerStatesTest { assertThat(task, shutdownTask(RecordProcessorCheckpointer.class, "recordProcessorCheckpointer", equalTo(recordProcessorCheckpointer))); assertThat(task, shutdownTask(ShutdownReason.class, "reason", equalTo(reason))); - assertThat(task, shutdownTask(LEASE_MANAGER_CLASS, "leaseManager", equalTo(leaseManager))); + assertThat(task, shutdownTask(LEASE_REFRESHER_CLASS, "leaseRefresher", equalTo(leaseRefresher))); assertThat(task, shutdownTask(InitialPositionInStreamExtended.class, "initialPositionInStream", equalTo(initialPositionInStream))); assertThat(task, diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/ProcessTaskTest.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/ProcessTaskTest.java index 807adcfc..e4dcd563 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/ProcessTaskTest.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/ProcessTaskTest.java @@ -47,7 +47,7 @@ import com.google.protobuf.ByteString; import lombok.Data; import software.amazon.kinesis.checkpoint.RecordProcessorCheckpointer; -import software.amazon.kinesis.leases.LeaseManagerProxy; +import software.amazon.kinesis.leases.ShardDetector; import software.amazon.kinesis.leases.ShardInfo; import software.amazon.kinesis.processor.RecordProcessor; import software.amazon.kinesis.retrieval.ThrottlingReporter; @@ -67,7 +67,7 @@ public class ProcessTaskTest { @Mock private ProcessRecordsInput processRecordsInput; @Mock - private LeaseManagerProxy leaseManagerProxy; + private ShardDetector shardDetector; @SuppressWarnings("serial") private static class RecordSubclass extends Record { @@ -95,7 +95,7 @@ public class ProcessTaskTest { private ProcessTask makeProcessTask(ProcessRecordsInput processRecordsInput) { return new ProcessTask(shardInfo, recordProcessor, checkpointer, taskBackoffTimeMillis, - skipShardSyncAtWorkerInitializationIfLeasesExist, leaseManagerProxy, throttlingReporter, + skipShardSyncAtWorkerInitializationIfLeasesExist, shardDetector, throttlingReporter, processRecordsInput, shouldCallProcessRecordsEvenForEmptyRecordList, IDLE_TIME_IN_MILLISECONDS); } diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/ShardConsumerTest.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/ShardConsumerTest.java index 7f340a1f..958e015d 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/ShardConsumerTest.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/ShardConsumerTest.java @@ -73,11 +73,10 @@ import com.amazonaws.services.kinesis.model.ShardIteratorType; import lombok.extern.slf4j.Slf4j; import software.amazon.kinesis.checkpoint.Checkpoint; -import software.amazon.kinesis.coordinator.KinesisClientLibConfiguration; import software.amazon.kinesis.checkpoint.RecordProcessorCheckpointer; -import software.amazon.kinesis.leases.LeaseManager; -import software.amazon.kinesis.leases.KinesisClientLease; -import software.amazon.kinesis.leases.LeaseManagerProxy; +import software.amazon.kinesis.coordinator.KinesisClientLibConfiguration; +import software.amazon.kinesis.leases.LeaseRefresher; +import software.amazon.kinesis.leases.ShardDetector; import software.amazon.kinesis.leases.ShardInfo; import software.amazon.kinesis.metrics.IMetricsFactory; import software.amazon.kinesis.metrics.NullMetricsFactory; @@ -139,7 +138,7 @@ public class ShardConsumerTest { @Mock private KinesisClientLibConfiguration config; @Mock - private LeaseManager leaseManager; + private LeaseRefresher leaseRefresher; @Mock private Checkpointer checkpoint; @Mock @@ -149,7 +148,7 @@ public class ShardConsumerTest { @Mock private RecordProcessorCheckpointer recordProcessorCheckpointer; @Mock - private LeaseManagerProxy leaseManagerProxy; + private ShardDetector shardDetector; @Before public void setup() { @@ -225,7 +224,7 @@ public class ShardConsumerTest { final ExtendedSequenceNumber checkpointSequenceNumber = new ExtendedSequenceNumber("123"); final ExtendedSequenceNumber pendingCheckpointSequenceNumber = null; - when(leaseManager.getLease(anyString())).thenReturn(null); + when(leaseRefresher.getLease(anyString())).thenReturn(null); when(checkpoint.getCheckpointObject(anyString())).thenReturn( new Checkpoint(checkpointSequenceNumber, pendingCheckpointSequenceNumber)); @@ -289,7 +288,7 @@ public class ShardConsumerTest { final int maxRecords = 2; Checkpointer checkpoint = new InMemoryCheckpointer(startSeqNum.toString()); checkpoint.setCheckpoint(shardId, ExtendedSequenceNumber.TRIM_HORIZON, concurrencyToken); - when(leaseManager.getLease(anyString())).thenReturn(null); + when(leaseRefresher.getLease(anyString())).thenReturn(null); TestStreamlet processor = new TestStreamlet(); shardInfo = new ShardInfo(shardId, concurrencyToken, null, null); @@ -392,7 +391,7 @@ public class ShardConsumerTest { final int maxRecords = 2; Checkpointer checkpoint = new InMemoryCheckpointer(startSeqNum.toString()); checkpoint.setCheckpoint(shardId, ExtendedSequenceNumber.TRIM_HORIZON, concurrencyToken); - when(leaseManager.getLease(anyString())).thenReturn(null); + when(leaseRefresher.getLease(anyString())).thenReturn(null); TransientShutdownErrorTestStreamlet processor = new TransientShutdownErrorTestStreamlet(); shardInfo = new ShardInfo(shardId, concurrencyToken, null, null); @@ -487,7 +486,7 @@ public class ShardConsumerTest { final int maxRecords = 2; Checkpointer checkpoint = new InMemoryCheckpointer(startSeqNum.toString()); checkpoint.setCheckpoint(streamShardId, ExtendedSequenceNumber.AT_TIMESTAMP, testConcurrencyToken); - when(leaseManager.getLease(anyString())).thenReturn(null); + when(leaseRefresher.getLease(anyString())).thenReturn(null); TestStreamlet processor = new TestStreamlet(); when(recordsFetcherFactory.createRecordsFetcher(any(GetRecordsRetrievalStrategy.class), anyString(), @@ -548,7 +547,7 @@ public class ShardConsumerTest { final ExtendedSequenceNumber checkpointSequenceNumber = new ExtendedSequenceNumber("123"); final ExtendedSequenceNumber pendingCheckpointSequenceNumber = new ExtendedSequenceNumber("999"); - when(leaseManager.getLease(anyString())).thenReturn(null); + when(leaseRefresher.getLease(anyString())).thenReturn(null); when(config.getRecordsFetcherFactory()).thenReturn(new SimpleRecordsFetcherFactory()); when(checkpoint.getCheckpointObject(anyString())).thenReturn( new Checkpoint(checkpointSequenceNumber, pendingCheckpointSequenceNumber)); @@ -642,7 +641,7 @@ public class ShardConsumerTest { final Optional logWarningForTaskAfterMillis) { return new ShardConsumer(shardInfo, streamName, - leaseManager, + leaseRefresher, executorService, getRecordsCache, recordProcessor, @@ -660,7 +659,7 @@ public class ShardConsumerTest { initialPositionLatest, cleanupLeasesOfCompletedShards, ignoreUnexpectedChildShards, - leaseManagerProxy, + shardDetector, metricsFactory); } diff --git a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/ShutdownTaskTest.java b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/ShutdownTaskTest.java index 62e38eb5..438655e8 100644 --- a/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/ShutdownTaskTest.java +++ b/amazon-kinesis-client/src/test/java/software/amazon/kinesis/lifecycle/ShutdownTaskTest.java @@ -34,9 +34,8 @@ import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionIn import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStreamExtended; import software.amazon.kinesis.checkpoint.RecordProcessorCheckpointer; -import software.amazon.kinesis.leases.LeaseManager; -import software.amazon.kinesis.leases.KinesisClientLease; -import software.amazon.kinesis.leases.LeaseManagerProxy; +import software.amazon.kinesis.leases.LeaseRefresher; +import software.amazon.kinesis.leases.ShardDetector; import software.amazon.kinesis.leases.ShardInfo; import software.amazon.kinesis.processor.RecordProcessor; import software.amazon.kinesis.retrieval.GetRecordsCache; @@ -66,9 +65,9 @@ public class ShutdownTaskTest { @Mock private RecordProcessorCheckpointer checkpointer; @Mock - private LeaseManager leaseManager; + private LeaseRefresher leaseRefresher; @Mock - private LeaseManagerProxy leaseManagerProxy; + private ShardDetector shardDetector; @Before public void setUp() throws Exception { @@ -78,9 +77,9 @@ public class ShutdownTaskTest { ExtendedSequenceNumber.LATEST); recordProcessor = new TestStreamlet(); - task = new ShutdownTask(shardInfo, leaseManagerProxy, recordProcessor, checkpointer, + task = new ShutdownTask(shardInfo, shardDetector, recordProcessor, checkpointer, TERMINATE_SHUTDOWN_REASON, INITIAL_POSITION_TRIM_HORIZON, cleanupLeasesOfCompletedShards, - ignoreUnexpectedChildShards, leaseManager, TASK_BACKOFF_TIME_MILLIS, getRecordsCache); + ignoreUnexpectedChildShards, leaseRefresher, TASK_BACKOFF_TIME_MILLIS, getRecordsCache); } /** @@ -100,7 +99,7 @@ public class ShutdownTaskTest { @Test public final void testCallWhenSyncingShardsThrows() { when(checkpointer.lastCheckpointValue()).thenReturn(ExtendedSequenceNumber.SHARD_END); - when(leaseManagerProxy.listShards()).thenReturn(null); + when(shardDetector.listShards()).thenReturn(null); TaskResult result = task.call(); assertNotNull(result.getException());