From 9cb5020022169cc368f4fcd1a56b5379d0fd25c6 Mon Sep 17 00:00:00 2001 From: Ashwin Giridharan Date: Fri, 22 Jan 2021 21:51:03 -0800 Subject: [PATCH 01/48] 1. Fix for updating HashRange directly while creating leases and including HashRange in seralizing and deserializing to/from DDB 2. Fix for making LeaseCleanupManager non-singleton to avoid cross-table interference in multiple apps running in same JVM 3. Fixing updateMetaInfo method to not update other lease table fields 4. Preventing shard deletion in LeaseCleanupManager if a valid shard does not have child shards in lease table and in Kinesis Service 5. Adding childshards update support in updateMetaInfo 6. Fixing LeaseCleanupManager to call updateMetaInfo instead of update for childshard update in lease 7. Fixing unit tests to accommodate HashRange changes --- .../lib/worker/KinesisShardSyncer.java | 5 ++- .../lib/worker/ShardConsumer.java | 2 +- .../clientlibrary/lib/worker/Worker.java | 8 ++-- .../impl/KinesisClientLeaseSerializer.java | 21 ++++++++- .../leases/impl/LeaseCleanupManager.java | 29 ++++++------ .../kinesis/leases/impl/LeaseManager.java | 1 - .../lib/worker/ShardSyncerTest.java | 44 +++++++++++-------- .../clientlibrary/lib/worker/WorkerTest.java | 4 ++ .../proxies/KinesisLocalFileProxy.java | 3 ++ .../util/KinesisLocalFileDataCreator.java | 1 + 10 files changed, 77 insertions(+), 41 deletions(-) diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisShardSyncer.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisShardSyncer.java index 4235bad9..af2b7c36 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisShardSyncer.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisShardSyncer.java @@ -45,6 +45,8 @@ import com.amazonaws.services.kinesis.metrics.impl.MetricsHelper; import com.amazonaws.services.kinesis.metrics.interfaces.MetricsLevel; import com.amazonaws.services.kinesis.model.Shard; +import static com.amazonaws.services.kinesis.leases.impl.HashKeyRangeForLease.fromHashKeyRange; + /** * Helper class to sync leases with shards of the Kinesis stream. * It will create new leases/activities when it discovers new Kinesis shards (bootstrap/resharding). @@ -617,7 +619,7 @@ class KinesisShardSyncer implements ShardSyncer { } newLease.setParentShardIds(parentShardIds); newLease.setOwnerSwitchesSinceCheckpoint(0L); - + newLease.setHashKeyRange(fromHashKeyRange(shard.getHashKeyRange())); return newLease; } @@ -641,6 +643,7 @@ class KinesisShardSyncer implements ShardSyncer { newLease.setParentShardIds(parentShardIds); newLease.setOwnerSwitchesSinceCheckpoint(0L); newLease.setCheckpoint(ExtendedSequenceNumber.TRIM_HORIZON); + newLease.setHashKeyRange(fromHashKeyRange(childShard.getHashKeyRange())); return newLease; } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumer.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumer.java index 29a95ac4..9e73aad9 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumer.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumer.java @@ -243,7 +243,7 @@ class ShardConsumer { this(shardInfo, streamConfig, checkpoint, recordProcessor, recordProcessorCheckpointer, leaseCoordinator, parentShardPollIntervalMillis, cleanupLeasesOfCompletedShards, executorService, metricsFactory, backoffTimeMillis, skipShardSyncAtWorkerInitializationIfLeasesExist, kinesisDataFetcher, retryGetRecordsInSeconds, - maxGetRecordsThreadPool, config, shardSyncer, shardSyncStrategy, LeaseCleanupManager.createOrGetInstance(streamConfig.getStreamProxy(), leaseCoordinator.getLeaseManager(), + maxGetRecordsThreadPool, config, shardSyncer, shardSyncStrategy, LeaseCleanupManager.newInstance(streamConfig.getStreamProxy(), leaseCoordinator.getLeaseManager(), Executors.newSingleThreadScheduledExecutor(), metricsFactory, config.shouldCleanupLeasesUponShardCompletion(), config.leaseCleanupIntervalMillis(), config.completedLeaseCleanupThresholdMillis(), config.garbageLeaseCleanupThresholdMillis(), config.getMaxRecords())); diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java index f8c66181..e7e725f8 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java @@ -96,9 +96,9 @@ public class Worker implements Runnable { // Default configs for periodic shard sync private static final int SHARD_SYNC_SLEEP_FOR_PERIODIC_SHARD_SYNC = 0; private static final int PERIODIC_SHARD_SYNC_MAX_WORKERS_DEFAULT = 1; //Default for KCL. - static final long LEASE_TABLE_CHECK_FREQUENCY_MILLIS = 3 * 1000L; - static final long MIN_WAIT_TIME_FOR_LEASE_TABLE_CHECK_MILLIS = 1 * 1000L; - static final long MAX_WAIT_TIME_FOR_LEASE_TABLE_CHECK_MILLIS = 30 * 1000L; + static long LEASE_TABLE_CHECK_FREQUENCY_MILLIS = 3 * 1000L; + static long MIN_WAIT_TIME_FOR_LEASE_TABLE_CHECK_MILLIS = 1 * 1000L; + static long MAX_WAIT_TIME_FOR_LEASE_TABLE_CHECK_MILLIS = 30 * 1000L; private static final WorkerStateChangeListener DEFAULT_WORKER_STATE_CHANGE_LISTENER = new NoOpWorkerStateChangeListener(); private static final LeaseCleanupValidator DEFAULT_LEASE_CLEANUP_VALIDATOR = new KinesisLeaseCleanupValidator(); @@ -576,7 +576,7 @@ public class Worker implements Runnable { this.workerStateChangeListener = workerStateChangeListener; workerStateChangeListener.onWorkerStateChange(WorkerStateChangeListener.WorkerState.CREATED); createShardSyncStrategy(config.getShardSyncStrategyType(), leaderDecider, periodicShardSyncManager); - this.leaseCleanupManager = LeaseCleanupManager.createOrGetInstance(streamConfig.getStreamProxy(), leaseCoordinator.getLeaseManager(), + this.leaseCleanupManager = LeaseCleanupManager.newInstance(streamConfig.getStreamProxy(), leaseCoordinator.getLeaseManager(), Executors.newSingleThreadScheduledExecutor(), metricsFactory, cleanupLeasesUponShardCompletion, config.leaseCleanupIntervalMillis(), config.completedLeaseCleanupThresholdMillis(), config.garbageLeaseCleanupThresholdMillis(), config.getMaxRecords()); diff --git a/src/main/java/com/amazonaws/services/kinesis/leases/impl/KinesisClientLeaseSerializer.java b/src/main/java/com/amazonaws/services/kinesis/leases/impl/KinesisClientLeaseSerializer.java index 310edb67..5a7fb9b2 100644 --- a/src/main/java/com/amazonaws/services/kinesis/leases/impl/KinesisClientLeaseSerializer.java +++ b/src/main/java/com/amazonaws/services/kinesis/leases/impl/KinesisClientLeaseSerializer.java @@ -68,6 +68,11 @@ public class KinesisClientLeaseSerializer implements ILeaseSerializer implements ILeaseManager { request.setExpected(serializer.getDynamoExistentExpectation(lease.getLeaseKey())); Map updates = serializer.getDynamoUpdateLeaseUpdate(lease, updateField); - updates.putAll(serializer.getDynamoUpdateLeaseUpdate(lease)); request.setAttributeUpdates(updates); try { diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncerTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncerTest.java index 9e9870d3..731cfdd3 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncerTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncerTest.java @@ -93,6 +93,8 @@ public class ShardSyncerTest { private LeaseManager leaseManager = new KinesisClientLeaseManager("tempTestTable", ddbClient, KinesisClientLibConfiguration.DEFAULT_DDB_BILLING_MODE); protected static final KinesisLeaseCleanupValidator leaseCleanupValidator = new KinesisLeaseCleanupValidator(); private static final KinesisShardSyncer shardSyncer = new KinesisShardSyncer(leaseCleanupValidator); + private static final HashKeyRange hashKeyRange = new HashKeyRange().withStartingHashKey("0").withEndingHashKey("10"); + /** * Old/Obsolete max value of a sequence number (2^128 -1). */ @@ -154,10 +156,10 @@ public class ShardSyncerTest { SequenceNumberRange sequenceRange = ShardObjectHelper.newSequenceNumberRange("342980", null); String shardId0 = "shardId-0"; - shards.add(ShardObjectHelper.newShard(shardId0, null, null, sequenceRange)); + shards.add(ShardObjectHelper.newShard(shardId0, null, null, sequenceRange, hashKeyRange)); String shardId1 = "shardId-1"; - shards.add(ShardObjectHelper.newShard(shardId1, null, null, sequenceRange)); + shards.add(ShardObjectHelper.newShard(shardId1, null, null, sequenceRange, hashKeyRange)); final LeaseSynchronizer leaseSynchronizer = getLeaseSynchronizer(shards, currentLeases); @@ -183,16 +185,16 @@ public class ShardSyncerTest { SequenceNumberRange sequenceRange = ShardObjectHelper.newSequenceNumberRange("342980", null); String shardId0 = "shardId-0"; - shards.add(ShardObjectHelper.newShard(shardId0, null, null, sequenceRange)); + shards.add(ShardObjectHelper.newShard(shardId0, null, null, sequenceRange, hashKeyRange)); String shardId1 = "shardId-1"; - shards.add(ShardObjectHelper.newShard(shardId1, null, null, sequenceRange)); + shards.add(ShardObjectHelper.newShard(shardId1, null, null, sequenceRange, hashKeyRange)); String shardId2 = "shardId-2"; - shards.add(ShardObjectHelper.newShard(shardId2, shardId1, null, sequenceRange)); + shards.add(ShardObjectHelper.newShard(shardId2, shardId1, null, sequenceRange, hashKeyRange)); String shardIdWithLease = "shardId-3"; - shards.add(ShardObjectHelper.newShard(shardIdWithLease, shardIdWithLease, null, sequenceRange)); + shards.add(ShardObjectHelper.newShard(shardIdWithLease, shardIdWithLease, null, sequenceRange, hashKeyRange)); currentLeases.add(newLease(shardIdWithLease)); @@ -699,9 +701,9 @@ public class ShardSyncerTest { SequenceNumberRange sequenceRange = ShardObjectHelper.newSequenceNumberRange("342980", null); String shardId0 = "shardId-0"; - shards.add(ShardObjectHelper.newShard(shardId0, null, null, sequenceRange)); + shards.add(ShardObjectHelper.newShard(shardId0, null, null, sequenceRange, hashKeyRange)); String shardId1 = "shardId-1"; - shards.add(ShardObjectHelper.newShard(shardId1, null, null, sequenceRange)); + shards.add(ShardObjectHelper.newShard(shardId1, null, null, sequenceRange, hashKeyRange)); File dataFile = KinesisLocalFileDataCreator.generateTempDataFile(shards, 2, "testBootstrap1"); dataFile.deleteOnExit(); IKinesisProxy kinesisProxy = new KinesisLocalFileProxy(dataFile.getAbsolutePath()); @@ -731,10 +733,10 @@ public class ShardSyncerTest { SequenceNumberRange sequenceRange = ShardObjectHelper.newSequenceNumberRange("342980", null); String shardId0 = "shardId-0"; - shards.add(ShardObjectHelper.newShard(shardId0, null, null, sequenceRange)); + shards.add(ShardObjectHelper.newShard(shardId0, null, null, sequenceRange, hashKeyRange)); String shardId1 = "shardId-1"; - shards.add(ShardObjectHelper.newShard(shardId1, null, null, sequenceRange)); + shards.add(ShardObjectHelper.newShard(shardId1, null, null, sequenceRange, hashKeyRange)); Set initialPositions = new HashSet(); initialPositions.add(INITIAL_POSITION_LATEST); @@ -769,17 +771,20 @@ public class ShardSyncerTest { shardsWithoutLeases.add(ShardObjectHelper.newShard("shardId-0", null, null, - ShardObjectHelper.newSequenceNumberRange("303", "404"))); + ShardObjectHelper.newSequenceNumberRange("303", "404"), + hashKeyRange)); final String lastShardId = "shardId-1"; shardsWithoutLeases.add(ShardObjectHelper.newShard(lastShardId, null, null, - ShardObjectHelper.newSequenceNumberRange("405", null))); + ShardObjectHelper.newSequenceNumberRange("405", null), + hashKeyRange)); shardsWithLeases.add(ShardObjectHelper.newShard("shardId-2", null, null, - ShardObjectHelper.newSequenceNumberRange("202", "302"))); + ShardObjectHelper.newSequenceNumberRange("202", "302"), + hashKeyRange)); currentLeases.add(newLease("shardId-2")); final List allShards = @@ -805,12 +810,14 @@ public class ShardSyncerTest { shards.add(ShardObjectHelper.newShard(firstShardId, null, null, - ShardObjectHelper.newSequenceNumberRange("303", "404"))); + ShardObjectHelper.newSequenceNumberRange("303", "404"), + hashKeyRange)); final String lastShardId = "shardId-1"; shards.add(ShardObjectHelper.newShard(lastShardId, null, null, - ShardObjectHelper.newSequenceNumberRange("405", null))); + ShardObjectHelper.newSequenceNumberRange("405", null), + hashKeyRange)); final LeaseSynchronizer leaseSynchronizer = getLeaseSynchronizer(shards, currentLeases); @@ -1969,14 +1976,14 @@ public class ShardSyncerTest { Map kinesisShards = new HashMap(); String parentShardId = "shardId-parent"; - kinesisShards.put(parentShardId, ShardObjectHelper.newShard(parentShardId, null, null, null)); + kinesisShards.put(parentShardId, ShardObjectHelper.newShard(parentShardId, null, null, null, hashKeyRange)); shardIdsOfCurrentLeases.add(parentShardId); String adjacentParentShardId = "shardId-adjacentParent"; - kinesisShards.put(adjacentParentShardId, ShardObjectHelper.newShard(adjacentParentShardId, null, null, null)); + kinesisShards.put(adjacentParentShardId, ShardObjectHelper.newShard(adjacentParentShardId, null, null, null, hashKeyRange)); String shardId = "shardId-9-1"; - Shard shard = ShardObjectHelper.newShard(shardId, parentShardId, adjacentParentShardId, null); + Shard shard = ShardObjectHelper.newShard(shardId, parentShardId, adjacentParentShardId, null, hashKeyRange); kinesisShards.put(shardId, shard); final MemoizationContext memoizationContext = new MemoizationContext(); @@ -2097,6 +2104,7 @@ public class ShardSyncerTest { String adjacentParentShardId = "shardId-adjacentParent"; shard.setParentShardId(parentShardId); shard.setAdjacentParentShardId(adjacentParentShardId); + shard.setHashKeyRange(hashKeyRange); KinesisClientLease lease = shardSyncer.newKCLLease(shard); Assert.assertEquals(shardId, lease.getLeaseKey()); diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/WorkerTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/WorkerTest.java index a7ca1151..50a3aa9b 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/WorkerTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/WorkerTest.java @@ -204,6 +204,10 @@ public class WorkerTest { when(config.getRecordsFetcherFactory()).thenReturn(recordsFetcherFactory); when(leaseCoordinator.getLeaseManager()).thenReturn(mock(ILeaseManager.class)); when(streamConfig.getStreamProxy()).thenReturn(kinesisProxy); + Worker.MIN_WAIT_TIME_FOR_LEASE_TABLE_CHECK_MILLIS = 10; + Worker.MAX_WAIT_TIME_FOR_LEASE_TABLE_CHECK_MILLIS = 50; + Worker.LEASE_TABLE_CHECK_FREQUENCY_MILLIS = 10; + } // CHECKSTYLE:IGNORE AnonInnerLengthCheck FOR NEXT 50 LINES diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisLocalFileProxy.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisLocalFileProxy.java index fe922520..1e3ff6de 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisLocalFileProxy.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisLocalFileProxy.java @@ -35,6 +35,7 @@ import java.util.Map; import java.util.Set; import com.amazonaws.services.kinesis.model.ChildShard; +import com.amazonaws.services.kinesis.model.HashKeyRange; import com.amazonaws.services.kinesis.model.ShardFilter; import com.amazonaws.util.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -408,11 +409,13 @@ public class KinesisLocalFileProxy implements IKinesisProxy { ChildShard leftChild = new ChildShard(); leftChild.setShardId("shardId-1"); leftChild.setParentShards(parentShards); + leftChild.setHashKeyRange(new HashKeyRange().withStartingHashKey("0").withEndingHashKey("10")); childShards.add(leftChild); ChildShard rightChild = new ChildShard(); rightChild.setShardId("shardId-2"); rightChild.setParentShards(parentShards); + rightChild.setHashKeyRange(new HashKeyRange().withStartingHashKey("11").withEndingHashKey(MAX_HASHKEY_VALUE.toString())); childShards.add(rightChild); return childShards; } diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/util/KinesisLocalFileDataCreator.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/util/KinesisLocalFileDataCreator.java index b13d0233..2baac1ca 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/util/KinesisLocalFileDataCreator.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/util/KinesisLocalFileDataCreator.java @@ -163,6 +163,7 @@ public class KinesisLocalFileDataCreator { HashKeyRange hashKeyRange = new HashKeyRange(); hashKeyRange.setStartingHashKey(hashKeyRangeStart.toString()); hashKeyRange.setEndingHashKey(hashKeyRangeEnd.toString()); + shard.setHashKeyRange(hashKeyRange); shards.add(shard); } From 76234d172c92ac46591611cdb7b9b1d3b0d4465e Mon Sep 17 00:00:00 2001 From: Ashwin Giridharan Date: Sun, 24 Jan 2021 16:10:25 -0800 Subject: [PATCH 02/48] Making LeaseCleanupManager more verbose about lease cleanup failures --- .../leases/impl/LeaseCleanupManager.java | 61 ++++++++++++------- 1 file changed, 40 insertions(+), 21 deletions(-) diff --git a/src/main/java/com/amazonaws/services/kinesis/leases/impl/LeaseCleanupManager.java b/src/main/java/com/amazonaws/services/kinesis/leases/impl/LeaseCleanupManager.java index 49a0ca36..f6ff8ee0 100644 --- a/src/main/java/com/amazonaws/services/kinesis/leases/impl/LeaseCleanupManager.java +++ b/src/main/java/com/amazonaws/services/kinesis/leases/impl/LeaseCleanupManager.java @@ -176,6 +176,7 @@ public class LeaseCleanupManager { boolean alreadyCheckedForGarbageCollection = false; boolean wereChildShardsPresent = false; boolean wasResourceNotFound = false; + String cleanupFailureReason = ""; try { if (cleanupLeasesUponShardCompletion && timeToCheckForCompletedShard) { @@ -184,53 +185,55 @@ public class LeaseCleanupManager { Set childShardKeys = leaseFromDDB.getChildShardIds(); if (CollectionUtils.isNullOrEmpty(childShardKeys)) { try { + // throws ResourceNotFoundException childShardKeys = getChildShardsFromService(shardInfo); - if (CollectionUtils.isNullOrEmpty(childShardKeys)) { LOG.error("No child shards returned from service for shard " + shardInfo.getShardId()); // If no children shard is found in DDB and from service, then do not delete the lease throw new InvalidStateException("No child shards found for this supposedly " + "closed shard in both local DDB and in service " + shardInfo.getShardId()); - } else { wereChildShardsPresent = true; updateLeaseWithChildShards(leasePendingDeletion, childShardKeys); } - } catch (ResourceNotFoundException e) { - throw e; } finally { + // We rely on resource presence in service for garbage collection. Since we already + // made a call to getChildShardsFromService we would be coming to know if the resource + // is present of not. In latter case, we would throw ResourceNotFoundException, which is + // handled in catch block. alreadyCheckedForGarbageCollection = true; } } else { wereChildShardsPresent = true; } try { - cleanedUpCompletedLease = cleanupLeaseForCompletedShard(lease, childShardKeys); + final CompletedShardResult completedShardResult = cleanupLeaseForCompletedShard(lease, childShardKeys); + cleanedUpCompletedLease = completedShardResult.cleanedUp(); + cleanupFailureReason = completedShardResult.failureMsg(); } catch (Exception e) { // Suppressing the exception here, so that we can attempt for garbage cleanup. - LOG.warn("Unable to cleanup lease for shard " + shardInfo.getShardId()); + LOG.warn("Unable to cleanup lease for shard " + shardInfo.getShardId() + " due to " + e.getMessage()); } } else { LOG.info("Lease not present in lease table while cleaning the shard " + shardInfo.getShardId()); cleanedUpCompletedLease = true; } } - - if (!alreadyCheckedForGarbageCollection && timeToCheckForGarbageShard) { - try { - wereChildShardsPresent = !CollectionUtils + if (!cleanedUpCompletedLease && !alreadyCheckedForGarbageCollection && timeToCheckForGarbageShard) { + // throws ResourceNotFoundException + wereChildShardsPresent = !CollectionUtils .isNullOrEmpty(getChildShardsFromService(shardInfo)); - } catch (ResourceNotFoundException e) { - throw e; - } } } catch (ResourceNotFoundException e) { wasResourceNotFound = true; cleanedUpGarbageLease = cleanupLeaseForGarbageShard(lease); + cleanupFailureReason = cleanedUpGarbageLease ? "" : "DDB Lease Deletion Failed"; + } catch (Exception e) { + LOG.warn("Unable to cleanup lease for shard " + shardInfo.getShardId() + " : " + e.getMessage()); + cleanupFailureReason = e.getMessage(); } - return new LeaseCleanupResult(cleanedUpCompletedLease, cleanedUpGarbageLease, wereChildShardsPresent, - wasResourceNotFound); + wasResourceNotFound, cleanupFailureReason); } private Set getChildShardsFromService(ShardInfo shardInfo) { @@ -238,12 +241,16 @@ public class LeaseCleanupManager { return kinesisProxy.get(iterator, maxRecords).getChildShards().stream().map(c -> c.getShardId()).collect(Collectors.toSet()); } - // A lease that ended with SHARD_END from ResourceNotFoundException is safe to delete if it no longer exists in the // stream (known explicitly from ResourceNotFound being thrown when processing this shard), private boolean cleanupLeaseForGarbageShard(KinesisClientLease lease) throws DependencyException, ProvisionedThroughputException, InvalidStateException { LOG.info("Deleting lease " + lease.getLeaseKey() + " as it is not present in the stream."); - leaseManager.deleteLease(lease); + try { + leaseManager.deleteLease(lease); + } catch (Exception e) { + LOG.warn("Lease deletion failed for " + lease.getLeaseKey() + " due to " + e.getMessage()); + return false; + } return true; } @@ -263,8 +270,9 @@ public class LeaseCleanupManager { // We should only be deleting the current shard's lease if // 1. All of its children are currently being processed, i.e their checkpoint is not TRIM_HORIZON or AT_TIMESTAMP. // 2. Its parent shard lease(s) have already been deleted. - private boolean cleanupLeaseForCompletedShard(KinesisClientLease lease, Set childShardLeaseKeys) + private CompletedShardResult cleanupLeaseForCompletedShard(KinesisClientLease lease, Set childShardLeaseKeys) throws DependencyException, ProvisionedThroughputException, InvalidStateException, IllegalStateException { + final Set processedChildShardLeaseKeys = new HashSet<>(); for (String childShardLeaseKey : childShardLeaseKeys) { @@ -280,14 +288,17 @@ public class LeaseCleanupManager { } } - if (!allParentShardLeasesDeleted(lease) || !Objects.equals(childShardLeaseKeys, processedChildShardLeaseKeys)) { - return false; + boolean parentShardsDeleted = allParentShardLeasesDeleted(lease); + boolean childrenStartedProcessing = Objects.equals(childShardLeaseKeys, processedChildShardLeaseKeys); + + if (!parentShardsDeleted || !childrenStartedProcessing) { + return new CompletedShardResult(false, !parentShardsDeleted ? "Parent shard(s) not deleted yet" : "Child shard(s) yet to begin processing"); } LOG.info("Deleting lease " + lease.getLeaseKey() + " as it has been completely processed and processing of child shard(s) has begun."); leaseManager.deleteLease(lease); - return true; + return new CompletedShardResult(true, ""); } private void updateLeaseWithChildShards(LeasePendingDeletion leasePendingDeletion, Set childShardKeys) @@ -363,9 +374,17 @@ public class LeaseCleanupManager { boolean cleanedUpGarbageLease; boolean wereChildShardsPresent; boolean wasResourceNotFound; + String cleanupFailureReason; public boolean leaseCleanedUp() { return cleanedUpCompletedLease | cleanedUpGarbageLease; } } + + @Value + @Accessors(fluent = true) + private static class CompletedShardResult { + boolean cleanedUp; + String failureMsg; + } } From ddfe75bb2d1f5e19ee639588243857fd42b89052 Mon Sep 17 00:00:00 2001 From: ashwing Date: Mon, 25 Jan 2021 17:27:01 -0800 Subject: [PATCH 03/48] Bugs in 1.14.0 --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index f936d602..ef9f6ecb 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,6 @@ +# Bugs in 1.14.0 +The KCL version 1.14.0 has durability risks when you run multiple KCL applications in same JVM. Also there can be a potential stale checkpoint issue blocking children shards from processing during resharding. We are releasing a fix in 1.14.1 version. ETA - 01/28/2021 + # Amazon Kinesis Client Library for Java [![Build Status](https://travis-ci.org/awslabs/amazon-kinesis-client.svg?branch=master)](https://travis-ci.org/awslabs/amazon-kinesis-client) ![BuildStatus](https://codebuild.us-west-2.amazonaws.com/badges?uuid=eyJlbmNyeXB0ZWREYXRhIjoiaWo4bDYyUkpWaG9ZTy9zeFVoaVlWbEwxazdicDJLcmZwUUpFWVVBM0ZueEJSeFIzNkhURzdVbUd6WUZHcGNxa3BEUzNrL0I5Nzc4NE9rbXhvdEpNdlFRPSIsIml2UGFyYW1ldGVyU3BlYyI6IlZDaVZJSTM1QW95bFRTQnYiLCJtYXRlcmlhbFNldFNlcmlhbCI6MX0%3D&branch=v1.x) From b2a747f3ea6c8c0b71ee28bf4ebee276f4baa001 Mon Sep 17 00:00:00 2001 From: Ashwin Giridharan Date: Mon, 25 Jan 2021 23:02:35 -0800 Subject: [PATCH 04/48] 1. Update the checkpoint with non SHARD_END sequence number only if the DDB sequence number is not SHARD_END. 2. Verify the shard end checkpointing by directly looking up the ddb lease entry --- .../lib/worker/ShutdownTask.java | 24 ++++++++++++++++--- .../impl/KinesisClientLeaseSerializer.java | 14 +++++++++++ .../kinesis/leases/impl/LeaseManager.java | 5 +++- .../kinesis/leases/impl/LeaseSerializer.java | 5 ++++ .../leases/interfaces/ILeaseSerializer.java | 8 +++++++ .../lib/worker/ShardConsumerTest.java | 9 +++++-- .../lib/worker/ShutdownTaskTest.java | 21 ++++++++++++---- 7 files changed, 75 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTask.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTask.java index 274aaaa1..07c9fff2 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTask.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTask.java @@ -222,11 +222,29 @@ class ShutdownTask implements ITask { .withCheckpointer(recordProcessorCheckpointer); recordProcessor.shutdown(shardEndShutdownInput); - final ExtendedSequenceNumber lastCheckpointValue = recordProcessorCheckpointer.getLastCheckpointValue(); + boolean successfullyCheckpointedShardEnd = false; - final boolean successfullyCheckpointedShardEnd = lastCheckpointValue.equals(ExtendedSequenceNumber.SHARD_END); + KinesisClientLease leaseFromDdb = null; + try { + leaseFromDdb = leaseCoordinator.getLeaseManager().getLease(shardInfo.getShardId()); + } catch (Exception e) { + LOG.error("Shard " + shardInfo.getShardId() + " : Unable to get lease entry for shard to verify shard end checkpointing.", e); + } - if ((lastCheckpointValue == null) || (!successfullyCheckpointedShardEnd)) { + if (leaseFromDdb != null && leaseFromDdb.getCheckpoint() != null) { + successfullyCheckpointedShardEnd = leaseFromDdb.getCheckpoint().equals(ExtendedSequenceNumber.SHARD_END); + final ExtendedSequenceNumber lastCheckpointValue = recordProcessorCheckpointer.getLastCheckpointValue(); + if (!leaseFromDdb.getCheckpoint().equals(lastCheckpointValue)) { + LOG.error("Shard " + shardInfo.getShardId() + + " : Checkpoint information mismatch between authoritative source and local cache. " + + "This does not affect the application flow, but cut a ticket to Kinesis when you see this. " + + "Authoritative entry : " + leaseFromDdb.getCheckpoint() + " Cache entry : " + lastCheckpointValue); + } + } else { + LOG.error("Shard " + shardInfo.getShardId() + " : No lease checkpoint entry for shard to verify shard end checkpointing. Lease Entry : " + leaseFromDdb); + } + + if (!successfullyCheckpointedShardEnd) { throw new IllegalArgumentException("Application didn't checkpoint at end of shard " + shardInfo.getShardId() + ". Application must checkpoint upon shutdown. " + "See IRecordProcessor.shutdown javadocs for more information."); diff --git a/src/main/java/com/amazonaws/services/kinesis/leases/impl/KinesisClientLeaseSerializer.java b/src/main/java/com/amazonaws/services/kinesis/leases/impl/KinesisClientLeaseSerializer.java index 5a7fb9b2..27568fc4 100644 --- a/src/main/java/com/amazonaws/services/kinesis/leases/impl/KinesisClientLeaseSerializer.java +++ b/src/main/java/com/amazonaws/services/kinesis/leases/impl/KinesisClientLeaseSerializer.java @@ -22,6 +22,7 @@ import com.amazonaws.services.dynamodbv2.model.AttributeAction; import com.amazonaws.services.dynamodbv2.model.AttributeDefinition; import com.amazonaws.services.dynamodbv2.model.AttributeValue; import com.amazonaws.services.dynamodbv2.model.AttributeValueUpdate; +import com.amazonaws.services.dynamodbv2.model.ComparisonOperator; import com.amazonaws.services.dynamodbv2.model.ExpectedAttributeValue; import com.amazonaws.services.dynamodbv2.model.KeySchemaElement; import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber; @@ -126,6 +127,19 @@ public class KinesisClientLeaseSerializer implements ILeaseSerializer getDynamoLeaseCheckpointExpectation(KinesisClientLease lease) { + Map result = baseSerializer.getDynamoLeaseCheckpointExpectation(lease); + ExpectedAttributeValue eav; + + if (!lease.getCheckpoint().equals(ExtendedSequenceNumber.SHARD_END)) { + eav = new ExpectedAttributeValue(DynamoUtils.createAttributeValue(ExtendedSequenceNumber.SHARD_END.getSequenceNumber())); + eav.setComparisonOperator(ComparisonOperator.NE); + result.put(CHECKPOINT_SEQUENCE_NUMBER_KEY, eav); + } + return result; + } + @Override public Map getDynamoNonexistantExpectation() { return baseSerializer.getDynamoNonexistantExpectation(); diff --git a/src/main/java/com/amazonaws/services/kinesis/leases/impl/LeaseManager.java b/src/main/java/com/amazonaws/services/kinesis/leases/impl/LeaseManager.java index d813e627..d9250c2c 100644 --- a/src/main/java/com/amazonaws/services/kinesis/leases/impl/LeaseManager.java +++ b/src/main/java/com/amazonaws/services/kinesis/leases/impl/LeaseManager.java @@ -20,6 +20,7 @@ import java.util.Map; import java.util.concurrent.TimeUnit; import com.amazonaws.services.dynamodbv2.model.BillingMode; +import com.amazonaws.services.dynamodbv2.model.ExpectedAttributeValue; import com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisClientLibConfiguration; import com.amazonaws.services.kinesis.leases.util.DynamoUtils; import org.apache.commons.logging.Log; @@ -582,7 +583,9 @@ public class LeaseManager implements ILeaseManager { UpdateItemRequest request = new UpdateItemRequest(); request.setTableName(table); request.setKey(serializer.getDynamoHashKey(lease)); - request.setExpected(serializer.getDynamoLeaseCounterExpectation(lease)); + Map expectations = serializer.getDynamoLeaseCounterExpectation(lease); + expectations.putAll(serializer.getDynamoLeaseCheckpointExpectation(lease)); + request.setExpected(expectations); Map updates = serializer.getDynamoLeaseCounterUpdate(lease); updates.putAll(serializer.getDynamoUpdateLeaseUpdate(lease)); diff --git a/src/main/java/com/amazonaws/services/kinesis/leases/impl/LeaseSerializer.java b/src/main/java/com/amazonaws/services/kinesis/leases/impl/LeaseSerializer.java index 85381560..adb7087a 100644 --- a/src/main/java/com/amazonaws/services/kinesis/leases/impl/LeaseSerializer.java +++ b/src/main/java/com/amazonaws/services/kinesis/leases/impl/LeaseSerializer.java @@ -127,6 +127,11 @@ public class LeaseSerializer implements ILeaseSerializer { return result; } + @Override + public Map getDynamoLeaseCheckpointExpectation(final Lease lease) { + return new HashMap<>(); + } + @Override public Map getDynamoNonexistantExpectation() { Map result = new HashMap(); diff --git a/src/main/java/com/amazonaws/services/kinesis/leases/interfaces/ILeaseSerializer.java b/src/main/java/com/amazonaws/services/kinesis/leases/interfaces/ILeaseSerializer.java index 2d9ea0c9..e69d1bf6 100644 --- a/src/main/java/com/amazonaws/services/kinesis/leases/interfaces/ILeaseSerializer.java +++ b/src/main/java/com/amazonaws/services/kinesis/leases/interfaces/ILeaseSerializer.java @@ -74,6 +74,14 @@ public interface ILeaseSerializer { */ public Map getDynamoLeaseOwnerExpectation(T lease); + /** + * @param lease + * @return the attribute value map asserting that the checkpoint state is as expected. + */ + default Map getDynamoLeaseCheckpointExpectation(T lease) { + throw new UnsupportedOperationException("DynamoLeaseCheckpointExpectation is not implemented"); + } + /** * @return the attribute value map asserting that a lease does not exist. */ diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumerTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumerTest.java index cb89b619..7afa3a8a 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumerTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumerTest.java @@ -587,7 +587,9 @@ public class ShardConsumerTest { parentShardIds.add("parentShardId"); KinesisClientLease currentLease = createLease(streamShardId, "leaseOwner", parentShardIds); currentLease.setCheckpoint(new ExtendedSequenceNumber("testSequenceNumbeer")); - when(leaseManager.getLease(streamShardId)).thenReturn(currentLease); + KinesisClientLease currentLease1 = createLease(streamShardId, "leaseOwner", parentShardIds); + currentLease1.setCheckpoint(ExtendedSequenceNumber.SHARD_END); + when(leaseManager.getLease(streamShardId)).thenReturn(currentLease, currentLease, currentLease1); when(leaseCoordinator.getCurrentlyHeldLease(shardInfo.getShardId())).thenReturn(currentLease); RecordProcessorCheckpointer recordProcessorCheckpointer = new RecordProcessorCheckpointer( @@ -714,7 +716,10 @@ public class ShardConsumerTest { parentShardIds.add("parentShardId"); KinesisClientLease currentLease = createLease(streamShardId, "leaseOwner", parentShardIds); currentLease.setCheckpoint(new ExtendedSequenceNumber("testSequenceNumbeer")); - when(leaseManager.getLease(streamShardId)).thenReturn(currentLease); + KinesisClientLease currentLease1 = createLease(streamShardId, "leaseOwner", parentShardIds); + currentLease1.setCheckpoint(ExtendedSequenceNumber.SHARD_END); + when(leaseManager.getLease(streamShardId)).thenReturn(currentLease, currentLease, currentLease1); + when(leaseCoordinator.getLeaseManager()).thenReturn(leaseManager); TransientShutdownErrorTestStreamlet processor = new TransientShutdownErrorTestStreamlet(); diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTaskTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTaskTest.java index 8b67f5dc..053a8bf7 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTaskTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTaskTest.java @@ -19,6 +19,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -211,11 +212,16 @@ public class ShutdownTaskTest { boolean cleanupLeasesOfCompletedShards = false; boolean ignoreUnexpectedChildShards = false; - KinesisClientLease currentLease = createLease(defaultShardId, "leaseOwner", Collections.emptyList()); - currentLease.setCheckpoint(new ExtendedSequenceNumber("3298")); + KinesisClientLease currentLease1 = createLease(defaultShardId, "leaseOwner", Collections.emptyList()); + currentLease1.setCheckpoint(new ExtendedSequenceNumber("3298")); + KinesisClientLease currentLease2 = createLease(defaultShardId, "leaseOwner", Collections.emptyList()); + currentLease2.setCheckpoint(ExtendedSequenceNumber.SHARD_END); + KinesisClientLease adjacentParentLease = createLease("ShardId-1", "leaseOwner", Collections.emptyList()); - when(leaseCoordinator.getCurrentlyHeldLease(defaultShardId)).thenReturn( currentLease); - when(leaseManager.getLease(defaultShardId)).thenReturn(currentLease); + when(leaseCoordinator.getCurrentlyHeldLease(defaultShardId)).thenReturn( currentLease1); + // 6 times as part of parent lease get in failure mode and then two times in actual execution + when(leaseManager.getLease(defaultShardId)).thenReturn(currentLease1, currentLease1, currentLease1, currentLease1, + currentLease1, currentLease1, currentLease1, currentLease2); when(leaseManager.getLease("ShardId-1")).thenReturn(null, null, null, null, null, adjacentParentLease); // Make first 5 attempts with partial parent info in lease table @@ -267,7 +273,7 @@ public class ShutdownTaskTest { verify(task, never()).isOneInNProbability(RETRY_RANDOM_MAX_RANGE); verify(getRecordsCache).shutdown(); verify(defaultRecordProcessor).shutdown(any(ShutdownInput.class)); - verify(leaseCoordinator, never()).dropLease(currentLease); + verify(leaseCoordinator, never()).dropLease(currentLease1); } @Test @@ -337,6 +343,11 @@ public class ShutdownTaskTest { public final void testCallWhenShardEnd() throws Exception { RecordProcessorCheckpointer checkpointer = mock(RecordProcessorCheckpointer.class); when(checkpointer.getLastCheckpointValue()).thenReturn(ExtendedSequenceNumber.SHARD_END); + final KinesisClientLease parentLease1 = createLease(defaultShardId, "leaseOwner", Collections.emptyList()); + parentLease1.setCheckpoint(new ExtendedSequenceNumber("3298")); + final KinesisClientLease parentLease2 = createLease(defaultShardId, "leaseOwner", Collections.emptyList()); + parentLease2.setCheckpoint(ExtendedSequenceNumber.SHARD_END); + when(leaseManager.getLease(defaultShardId)).thenReturn(parentLease1).thenReturn(parentLease2); boolean cleanupLeasesOfCompletedShards = false; boolean ignoreUnexpectedChildShards = false; From 03e162b770311617e0369632e006180a7f9bd080 Mon Sep 17 00:00:00 2001 From: Ashwin Giridharan Date: Mon, 25 Jan 2021 23:41:23 -0800 Subject: [PATCH 05/48] Improving LeaseCleanupManager's cleanupFailureReason --- .../services/kinesis/leases/impl/LeaseCleanupManager.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/com/amazonaws/services/kinesis/leases/impl/LeaseCleanupManager.java b/src/main/java/com/amazonaws/services/kinesis/leases/impl/LeaseCleanupManager.java index f6ff8ee0..16f5f353 100644 --- a/src/main/java/com/amazonaws/services/kinesis/leases/impl/LeaseCleanupManager.java +++ b/src/main/java/com/amazonaws/services/kinesis/leases/impl/LeaseCleanupManager.java @@ -218,6 +218,8 @@ public class LeaseCleanupManager { LOG.info("Lease not present in lease table while cleaning the shard " + shardInfo.getShardId()); cleanedUpCompletedLease = true; } + } else { + cleanupFailureReason = "Configuration/Interval condition not satisfied to execute lease cleanup this cycle"; } if (!cleanedUpCompletedLease && !alreadyCheckedForGarbageCollection && timeToCheckForGarbageShard) { // throws ResourceNotFoundException From c935bc403245b29edd5785e64420d48cf29697e1 Mon Sep 17 00:00:00 2001 From: ashwing Date: Tue, 26 Jan 2021 09:26:36 -0800 Subject: [PATCH 06/48] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ef9f6ecb..63040b42 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # Bugs in 1.14.0 -The KCL version 1.14.0 has durability risks when you run multiple KCL applications in same JVM. Also there can be a potential stale checkpoint issue blocking children shards from processing during resharding. We are releasing a fix in 1.14.1 version. ETA - 01/28/2021 +The KCL version 1.14.0 has cross table interference risks when you run multiple KCL applications in same JVM. Also there can be a potential stale checkpoint issue blocking children shards from processing during resharding. We are releasing a fix in 1.14.1 version. ETA - 01/28/2021 # Amazon Kinesis Client Library for Java [![Build Status](https://travis-ci.org/awslabs/amazon-kinesis-client.svg?branch=master)](https://travis-ci.org/awslabs/amazon-kinesis-client) ![BuildStatus](https://codebuild.us-west-2.amazonaws.com/badges?uuid=eyJlbmNyeXB0ZWREYXRhIjoiaWo4bDYyUkpWaG9ZTy9zeFVoaVlWbEwxazdicDJLcmZwUUpFWVVBM0ZueEJSeFIzNkhURzdVbUd6WUZHcGNxa3BEUzNrL0I5Nzc4NE9rbXhvdEpNdlFRPSIsIml2UGFyYW1ldGVyU3BlYyI6IlZDaVZJSTM1QW95bFRTQnYiLCJtYXRlcmlhbFNldFNlcmlhbCI6MX0%3D&branch=v1.x) From 530fd477b120e9611bf0da1ab770712b7f0c6a9d Mon Sep 17 00:00:00 2001 From: Ashwin Giridharan Date: Tue, 26 Jan 2021 15:18:29 -0800 Subject: [PATCH 07/48] Fixing record processor processing deleted leases in cycle --- .../clientlibrary/lib/worker/InitializeTask.java | 11 ++++++++++- .../clientlibrary/lib/worker/ShardConsumer.java | 10 +++++++++- .../kinesis/clientlibrary/lib/worker/TaskResult.java | 10 ++++++++++ 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/InitializeTask.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/InitializeTask.java index 21d0599b..474a5767 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/InitializeTask.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/InitializeTask.java @@ -14,6 +14,7 @@ */ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; +import com.amazonaws.services.kinesis.clientlibrary.exceptions.KinesisClientLibException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -79,7 +80,15 @@ class InitializeTask implements ITask { try { LOG.debug("Initializing ShardId " + shardInfo.getShardId()); - Checkpoint initialCheckpointObject = checkpoint.getCheckpointObject(shardInfo.getShardId()); + Checkpoint initialCheckpointObject; + try { + initialCheckpointObject = checkpoint.getCheckpointObject(shardInfo.getShardId()); + } catch (KinesisClientLibException e) { + LOG.error("Caught exception while fetching checkpoint for " + shardInfo.getShardId(), e); + final TaskResult result = new TaskResult(e); + result.leaseNotFound(); + return result; + } ExtendedSequenceNumber initialCheckpoint = initialCheckpointObject.getCheckpoint(); dataFetcher.initialize(initialCheckpoint.getSequenceNumber(), streamConfig.getInitialPositionInStream()); diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumer.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumer.java index 9e73aad9..11ee39b5 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumer.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumer.java @@ -374,7 +374,7 @@ class ShardConsumer { } private enum TaskOutcome { - SUCCESSFUL, END_OF_SHARD, NOT_COMPLETE, FAILURE + SUCCESSFUL, END_OF_SHARD, NOT_COMPLETE, FAILURE, LEASE_NOT_FOUND } private TaskOutcome determineTaskOutcome() { @@ -391,6 +391,10 @@ class ShardConsumer { return TaskOutcome.SUCCESSFUL; } logTaskException(result); + // This is the case of result with exception + if (result.isLeaseNotFound()) { + return TaskOutcome.LEASE_NOT_FOUND; + } } catch (Exception e) { throw new RuntimeException(e); } finally { @@ -487,6 +491,10 @@ class ShardConsumer { markForShutdown(ShutdownReason.TERMINATE); LOG.info("Shard " + shardInfo.getShardId() + ": Mark for shutdown with reason TERMINATE"); } + if (taskOutcome == TaskOutcome.LEASE_NOT_FOUND) { + markForShutdown(ShutdownReason.ZOMBIE); + LOG.info("Shard " + shardInfo.getShardId() + ": Mark for shutdown with reason ZOMBIE as lease was not found"); + } if (isShutdownRequested() && taskOutcome != TaskOutcome.FAILURE) { currentState = currentState.shutdownTransition(shutdownReason); } else if (isShutdownRequested() && ConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS.equals(currentState.getState())) { diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/TaskResult.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/TaskResult.java index 70109b86..db22c97b 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/TaskResult.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/TaskResult.java @@ -33,6 +33,8 @@ class TaskResult { // List of childShards of the current shard. This field is only required for the task result when we reach end of a shard. private List childShards; + private boolean leaseNotFound; + /** * @return the shardEndReached */ @@ -57,6 +59,14 @@ class TaskResult { */ protected void setChildShards(List childShards) { this.childShards = childShards; } + public boolean isLeaseNotFound() { + return leaseNotFound; + } + + public void leaseNotFound() { + this.leaseNotFound = true; + } + /** * @return the exception */ From 5142ce9bc512a0dcf6e935eb722dbf2b03aa9ef9 Mon Sep 17 00:00:00 2001 From: Ashwin Giridharan Date: Tue, 26 Jan 2021 21:40:22 -0800 Subject: [PATCH 08/48] Fixing exception type --- .../lib/worker/InitializeTask.java | 4 +- .../KinesisClientLibLeaseCoordinator.java | 3 +- .../lib/worker/ShardConsumerTest.java | 49 +++++++++++++++++-- 3 files changed, 50 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/InitializeTask.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/InitializeTask.java index 474a5767..5343470f 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/InitializeTask.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/InitializeTask.java @@ -14,7 +14,7 @@ */ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; -import com.amazonaws.services.kinesis.clientlibrary.exceptions.KinesisClientLibException; +import com.amazonaws.services.kinesis.clientlibrary.exceptions.KinesisClientLibNonRetryableException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -83,7 +83,7 @@ class InitializeTask implements ITask { Checkpoint initialCheckpointObject; try { initialCheckpointObject = checkpoint.getCheckpointObject(shardInfo.getShardId()); - } catch (KinesisClientLibException e) { + } catch (KinesisClientLibNonRetryableException e) { LOG.error("Caught exception while fetching checkpoint for " + shardInfo.getShardId(), e); final TaskResult result = new TaskResult(e); result.leaseNotFound(); diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibLeaseCoordinator.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibLeaseCoordinator.java index bcbabcde..47baad04 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibLeaseCoordinator.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibLeaseCoordinator.java @@ -283,7 +283,8 @@ class KinesisClientLibLeaseCoordinator extends LeaseCoordinator Date: Tue, 26 Jan 2021 23:21:39 -0800 Subject: [PATCH 09/48] Fixing unit test --- .../lib/worker/KinesisClientLibLeaseCoordinatorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibLeaseCoordinatorTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibLeaseCoordinatorTest.java index d8549870..e48cdd16 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibLeaseCoordinatorTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibLeaseCoordinatorTest.java @@ -79,7 +79,7 @@ public class KinesisClientLibLeaseCoordinatorTest { leaseCoordinator.initialize(); } - @Test(expected = KinesisClientLibIOException.class) + @Test(expected = com.amazonaws.services.kinesis.clientlibrary.exceptions.InvalidStateException.class) public void testGetCheckpointObjectWithNoLease() throws DependencyException, ProvisionedThroughputException, IllegalStateException, InvalidStateException, KinesisClientLibException { From 4f6f4e5971e09af0dd08dc4d9165740f1ac95dea Mon Sep 17 00:00:00 2001 From: Ashwin Giridharan Date: Wed, 27 Jan 2021 14:53:54 -0800 Subject: [PATCH 10/48] KCL 1.14.1 release --- CHANGELOG.md | 11 ++++++++++- README.md | 14 +++++++++++--- pom.xml | 2 +- .../lib/worker/KinesisClientLibConfiguration.java | 2 +- 4 files changed, 23 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3188791a..9fcc1bc0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,15 @@ # Changelog -## Latest Release (1.14.0 - August 17, 2020) +## Latest Release (1.14.1 - January 27, 2021) +* [Milestone#56](https://github.com/awslabs/amazon-kinesis-client/milestone/56) + +* Fix for cross DDB table interference when multiple KCL applications are run in same JVM. +* Fix and guards to avoid potential checkpoint rewind during shard end, which may block children shard processing. +* Fix for thread cycle wastage on InitializeTask for deleted shard +* Improved logging in LeaseCleanupManager that would indicate why certain shards are not cleaned up. + + +## Release (1.14.0 - August 17, 2020) * [Milestone#50](https://github.com/awslabs/amazon-kinesis-client/milestone/50) diff --git a/README.md b/README.md index 63040b42..e5452951 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -# Bugs in 1.14.0 -The KCL version 1.14.0 has cross table interference risks when you run multiple KCL applications in same JVM. Also there can be a potential stale checkpoint issue blocking children shards from processing during resharding. We are releasing a fix in 1.14.1 version. ETA - 01/28/2021 +# Bugs in 1.14.0 version +We recommend customers to migrate to 1.14.1 to avoid [known bugs](https://github.com/awslabs/amazon-kinesis-client/issues/778) in 1.14.0 version # Amazon Kinesis Client Library for Java [![Build Status](https://travis-ci.org/awslabs/amazon-kinesis-client.svg?branch=master)](https://travis-ci.org/awslabs/amazon-kinesis-client) ![BuildStatus](https://codebuild.us-west-2.amazonaws.com/badges?uuid=eyJlbmNyeXB0ZWREYXRhIjoiaWo4bDYyUkpWaG9ZTy9zeFVoaVlWbEwxazdicDJLcmZwUUpFWVVBM0ZueEJSeFIzNkhURzdVbUd6WUZHcGNxa3BEUzNrL0I5Nzc4NE9rbXhvdEpNdlFRPSIsIml2UGFyYW1ldGVyU3BlYyI6IlZDaVZJSTM1QW95bFRTQnYiLCJtYXRlcmlhbFNldFNlcmlhbCI6MX0%3D&branch=v1.x) @@ -34,7 +34,15 @@ To make it easier for developers to write record processors in other languages, ## Release Notes -### Latest Release (1.14.0 - August 17, 2020) +### Latest Release (1.14.1 - January 27, 2021) +* [Milestone#56](https://github.com/awslabs/amazon-kinesis-client/milestone/56) + +* Fix for cross DDB table interference when multiple KCL applications are run in same JVM. +* Fix and guards to avoid potential checkpoint rewind during shard end, which may block children shard processing. +* Fix for thread cycle wastage on InitializeTask for deleted shard. +* Improved logging in LeaseCleanupManager that would indicate why certain shards are not cleaned up. + +### Release (1.14.0 - August 17, 2020) * [Milestone#50](https://github.com/awslabs/amazon-kinesis-client/milestone/50) diff --git a/pom.xml b/pom.xml index 48a802ae..5042fd9e 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ amazon-kinesis-client jar Amazon Kinesis Client Library for Java - 1.14.0 + 1.14.1 The Amazon Kinesis Client Library for Java enables Java developers to easily consume and process data from Amazon Kinesis. diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java index 643b45d0..06fad2af 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java @@ -147,7 +147,7 @@ public class KinesisClientLibConfiguration { /** * User agent set when Amazon Kinesis Client Library makes AWS requests. */ - public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.14.0"; + public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.14.1"; /** * KCL will validate client provided sequence numbers with a call to Amazon Kinesis before checkpointing for calls From 8ba93e6aeedfa8b1d929bbbe658b26b1c1f3745c Mon Sep 17 00:00:00 2001 From: Ashwin Giridharan Date: Wed, 27 Jan 2021 16:08:41 -0800 Subject: [PATCH 11/48] Release notes update --- CHANGELOG.md | 4 ++-- README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fcc1bc0..3aad3a09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,8 +5,8 @@ * Fix for cross DDB table interference when multiple KCL applications are run in same JVM. * Fix and guards to avoid potential checkpoint rewind during shard end, which may block children shard processing. -* Fix for thread cycle wastage on InitializeTask for deleted shard -* Improved logging in LeaseCleanupManager that would indicate why certain shards are not cleaned up. +* Fix for thread cycle wastage on InitializeTask for deleted shard. +* Improved logging in LeaseCleanupManager that would indicate why certain shards are not cleaned up from the lease table. ## Release (1.14.0 - August 17, 2020) diff --git a/README.md b/README.md index e5452951..3fbfcb25 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ To make it easier for developers to write record processors in other languages, * Fix for cross DDB table interference when multiple KCL applications are run in same JVM. * Fix and guards to avoid potential checkpoint rewind during shard end, which may block children shard processing. * Fix for thread cycle wastage on InitializeTask for deleted shard. -* Improved logging in LeaseCleanupManager that would indicate why certain shards are not cleaned up. +* Improved logging in LeaseCleanupManager that would indicate why certain shards are not cleaned up from the lease table. ### Release (1.14.0 - August 17, 2020) From a13ed13702e27e200a9c8e079c66f486034b0a03 Mon Sep 17 00:00:00 2001 From: Joshua Kim <20001595+joshua-kim@users.noreply.github.com> Date: Wed, 24 Feb 2021 13:34:39 -0800 Subject: [PATCH 12/48] Fixing bug where ShardFilter parameter for ListShards was being passed in for paginated calls. (#790) Co-authored-by: Joshua Kim --- .../services/kinesis/clientlibrary/proxies/KinesisProxy.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxy.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxy.java index 0936bddb..c1d7f10d 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxy.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxy.java @@ -315,13 +315,12 @@ public class KinesisProxy implements IKinesisProxyExtended { request.setRequestCredentials(credentialsProvider.getCredentials()); if (StringUtils.isEmpty(nextToken)) { request.setStreamName(streamName); + request.setShardFilter(shardFilter); } else { request.setNextToken(nextToken); } - if (shardFilter != null) { - request.setShardFilter(shardFilter); - } + LOG.info("Listing shards with list shards request " + request); ListShardsResult result = null; LimitExceededException lastException = null; From f38dd18ed1d41b9dd3839366d16a58b832292a0b Mon Sep 17 00:00:00 2001 From: Joshua Kim <20001595+joshua-kim@users.noreply.github.com> Date: Wed, 24 Feb 2021 14:39:04 -0800 Subject: [PATCH 13/48] Preparing for 1.14.2 release (#793) Co-authored-by: Joshua Kim --- CHANGELOG.md | 7 ++++++- README.md | 6 +++++- pom.xml | 2 +- .../lib/worker/KinesisClientLibConfiguration.java | 2 +- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3aad3a09..b3e5ce41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ # Changelog -## Latest Release (1.14.1 - January 27, 2021) + +## Latest Release (1.14.2 - February 24, 2021) +* [Milestone#57](https://github.com/awslabs/amazon-kinesis-client/milestone/57) +* [#790](https://github.com/awslabs/amazon-kinesis-client/pull/790) Fixing a bug that caused paginated `ListShards` calls with the `ShardFilter` parameter to fail when the lease table was being initialized. + +## Release (1.14.1 - January 27, 2021) * [Milestone#56](https://github.com/awslabs/amazon-kinesis-client/milestone/56) * Fix for cross DDB table interference when multiple KCL applications are run in same JVM. diff --git a/README.md b/README.md index 3fbfcb25..4d3ca0fd 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,11 @@ To make it easier for developers to write record processors in other languages, ## Release Notes -### Latest Release (1.14.1 - January 27, 2021) +## Latest Release (1.14.2 - February 24, 2021) +* [Milestone#57](https://github.com/awslabs/amazon-kinesis-client/milestone/57) +* [#790](https://github.com/awslabs/amazon-kinesis-client/pull/790) Fixing a bug that caused paginated `ListShards` calls with the `ShardFilter` parameter to fail when the lease table was being initialized. + +### Release (1.14.1 - January 27, 2021) * [Milestone#56](https://github.com/awslabs/amazon-kinesis-client/milestone/56) * Fix for cross DDB table interference when multiple KCL applications are run in same JVM. diff --git a/pom.xml b/pom.xml index 5042fd9e..e4064be9 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ amazon-kinesis-client jar Amazon Kinesis Client Library for Java - 1.14.1 + 1.14.2 The Amazon Kinesis Client Library for Java enables Java developers to easily consume and process data from Amazon Kinesis. diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java index 06fad2af..cc385fd2 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java @@ -147,7 +147,7 @@ public class KinesisClientLibConfiguration { /** * User agent set when Amazon Kinesis Client Library makes AWS requests. */ - public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.14.1"; + public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.14.2"; /** * KCL will validate client provided sequence numbers with a call to Amazon Kinesis before checkpointing for calls From f2b9006a98d454d644794f88be02ec7187adc818 Mon Sep 17 00:00:00 2001 From: Joshua Kim <20001595+joshua-kim@users.noreply.github.com> Date: Mon, 3 May 2021 13:50:54 -0700 Subject: [PATCH 14/48] Removing cached shard progress, adding guardrails for duplicate shard responses. (#811) Co-authored-by: Joshua Kim --- .../clientlibrary/proxies/KinesisProxy.java | 60 +++++------- .../clientlibrary/utils/RequestUtil.java | 24 +++++ .../proxies/KinesisProxyTest.java | 91 +++++++++---------- 3 files changed, 90 insertions(+), 85 deletions(-) create mode 100644 src/main/java/com/amazonaws/services/kinesis/clientlibrary/utils/RequestUtil.java diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxy.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxy.java index c1d7f10d..c6b641bb 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxy.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxy.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.Date; import java.util.EnumSet; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -29,6 +30,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import java.util.stream.Collectors; +import com.amazonaws.services.kinesis.clientlibrary.utils.RequestUtil; import com.amazonaws.services.kinesis.model.ShardFilter; import com.amazonaws.util.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -59,7 +61,6 @@ import com.amazonaws.services.kinesis.model.ShardIteratorType; import com.amazonaws.services.kinesis.model.StreamStatus; import lombok.AccessLevel; -import lombok.Data; import lombok.Getter; import lombok.Setter; @@ -82,8 +83,6 @@ public class KinesisProxy implements IKinesisProxyExtended { private AmazonKinesis client; private AWSCredentialsProvider credentialsProvider; - private ShardIterationState shardIterationState = null; - @Setter(AccessLevel.PACKAGE) private volatile Map cachedShardMap = null; @Setter(AccessLevel.PACKAGE) @@ -442,10 +441,8 @@ public class KinesisProxy implements IKinesisProxyExtended { */ @Override public synchronized List getShardListWithFilter(ShardFilter shardFilter) { - if (shardIterationState == null) { - shardIterationState = new ShardIterationState(); - } - + final List shards = new ArrayList<>(); + final List requestIds = new ArrayList<>(); if (isKinesisClient) { ListShardsResult result; String nextToken = null; @@ -460,16 +457,18 @@ public class KinesisProxy implements IKinesisProxyExtended { */ return null; } else { - shardIterationState.update(result.getShards()); + shards.addAll(result.getShards()); + requestIds.add(RequestUtil.requestId(result)); nextToken = result.getNextToken(); } } while (StringUtils.isNotEmpty(result.getNextToken())); } else { DescribeStreamResult response; + String lastShardId = null; do { - response = getStreamInfo(shardIterationState.getLastShardId()); + response = getStreamInfo(lastShardId); if (response == null) { /* @@ -478,16 +477,26 @@ public class KinesisProxy implements IKinesisProxyExtended { */ return null; } else { - shardIterationState.update(response.getStreamDescription().getShards()); + final List pageOfShards = response.getStreamDescription().getShards(); + shards.addAll(pageOfShards); + requestIds.add(RequestUtil.requestId(response)); + + final Shard lastShard = pageOfShards.get(pageOfShards.size() - 1); + if (lastShardId == null || lastShardId.compareTo(lastShard.getShardId()) < 0) { + lastShardId = lastShard.getShardId(); + } } } while (response.getStreamDescription().isHasMoreShards()); } - List shards = shardIterationState.getShards(); - this.cachedShardMap = shards.stream().collect(Collectors.toMap(Shard::getShardId, Function.identity())); + final List dedupedShards = new ArrayList<>(new LinkedHashSet<>(shards)); + if (dedupedShards.size() < shards.size()) { + LOG.warn("Found duplicate shards in response when sync'ing from Kinesis. " + + "Request ids - " + requestIds + ". Response - " + shards); + } + this.cachedShardMap = dedupedShards.stream().collect(Collectors.toMap(Shard::getShardId, Function.identity())); this.lastCacheUpdateTime = Instant.now(); - shardIterationState = new ShardIterationState(); - return shards; + return dedupedShards; } /** @@ -617,27 +626,4 @@ public class KinesisProxy implements IKinesisProxyExtended { final PutRecordResult response = client.putRecord(putRecordRequest); return response; } - - @Data - static class ShardIterationState { - - private List shards; - private String lastShardId; - - public ShardIterationState() { - shards = new ArrayList<>(); - } - - public void update(List shards) { - if (shards == null || shards.isEmpty()) { - return; - } - this.shards.addAll(shards); - Shard lastShard = shards.get(shards.size() - 1); - if (lastShardId == null || lastShardId.compareTo(lastShard.getShardId()) < 0) { - lastShardId = lastShard.getShardId(); - } - } - } - } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/utils/RequestUtil.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/utils/RequestUtil.java new file mode 100644 index 00000000..cac65d45 --- /dev/null +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/utils/RequestUtil.java @@ -0,0 +1,24 @@ +package com.amazonaws.services.kinesis.clientlibrary.utils; + +import com.amazonaws.AmazonWebServiceResult; + +/** + * Helper class to parse metadata from AWS requests. + */ +public class RequestUtil { + private static final String DEFAULT_REQUEST_ID = "NONE"; + + /** + * Get the requestId associated with a request. + * + * @param result + * @return the requestId for a request, or "NONE" if one is not available. + */ + public static String requestId(AmazonWebServiceResult result) { + if (result == null || result.getSdkResponseMetadata() == null || result.getSdkResponseMetadata().getRequestId() == null) { + return DEFAULT_REQUEST_ID; + } + + return result.getSdkResponseMetadata().getRequestId(); + } +} diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxyTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxyTest.java index 75feb19e..76671176 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxyTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxyTest.java @@ -24,6 +24,7 @@ import static org.hamcrest.Matchers.nullValue; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; import static org.mockito.Matchers.any; import static org.mockito.Matchers.argThat; import static org.mockito.Mockito.doReturn; @@ -92,6 +93,7 @@ public class KinesisProxyTest { private static final String SHARD_4 = "shard-4"; private static final String NOT_CACHED_SHARD = "ShardId-0005"; private static final String NEVER_PRESENT_SHARD = "ShardId-0010"; + private static final String REQUEST_ID = "requestId"; @Mock private AmazonKinesis mockClient; @@ -249,54 +251,6 @@ public class KinesisProxyTest { ddbProxy.getShardList(); } - @Test - public void testGetStreamInfoStoresOffset() throws Exception { - when(describeStreamResult.getStreamDescription()).thenReturn(streamDescription); - when(streamDescription.getStreamStatus()).thenReturn(StreamStatus.ACTIVE.name()); - Shard shard1 = mock(Shard.class); - Shard shard2 = mock(Shard.class); - Shard shard3 = mock(Shard.class); - List shardList1 = Collections.singletonList(shard1); - List shardList2 = Collections.singletonList(shard2); - List shardList3 = Collections.singletonList(shard3); - - String shardId1 = "ShardId-0001"; - String shardId2 = "ShardId-0002"; - String shardId3 = "ShardId-0003"; - - when(shard1.getShardId()).thenReturn(shardId1); - when(shard2.getShardId()).thenReturn(shardId2); - when(shard3.getShardId()).thenReturn(shardId3); - - when(streamDescription.getShards()).thenReturn(shardList1).thenReturn(shardList2).thenReturn(shardList3); - when(streamDescription.isHasMoreShards()).thenReturn(true, true, false); - when(mockDDBStreamClient.describeStream(argThat(describeWithoutShardId()))).thenReturn(describeStreamResult); - - when(mockDDBStreamClient.describeStream(argThat(describeWithShardId(shardId1)))) - .thenThrow(new LimitExceededException("1"), new LimitExceededException("2"), - new LimitExceededException("3")) - .thenReturn(describeStreamResult); - - when(mockDDBStreamClient.describeStream(argThat(describeWithShardId(shardId2)))).thenReturn(describeStreamResult); - - boolean limitExceeded = false; - try { - ddbProxy.getShardList(); - } catch (LimitExceededException le) { - limitExceeded = true; - } - assertThat(limitExceeded, equalTo(true)); - List actualShards = ddbProxy.getShardList(); - List expectedShards = Arrays.asList(shard1, shard2, shard3); - - assertThat(actualShards, equalTo(expectedShards)); - - verify(mockDDBStreamClient).describeStream(argThat(describeWithoutShardId())); - verify(mockDDBStreamClient, times(4)).describeStream(argThat(describeWithShardId(shardId1))); - verify(mockDDBStreamClient).describeStream(argThat(describeWithShardId(shardId2))); - - } - @Test public void testListShardsWithMoreDataAvailable() { ListShardsResult responseWithMoreData = new ListShardsResult().withShards(shards.subList(0, 2)).withNextToken(NEXT_TOKEN); @@ -483,6 +437,47 @@ public class KinesisProxyTest { verify(mockClient).listShards(any()); } + /** + * Tests that if we fail halfway through a listShards call, we fail gracefully and subsequent calls are not + * affected by the failure of the first request. + */ + @Test + public void testNoDuplicateShardsInPartialFailure() { + proxy.setCachedShardMap(null); + + ListShardsResult firstPage = new ListShardsResult().withShards(shards.subList(0, 2)).withNextToken(NEXT_TOKEN); + ListShardsResult lastPage = new ListShardsResult().withShards(shards.subList(2, shards.size())).withNextToken(null); + + when(mockClient.listShards(any())) + .thenReturn(firstPage).thenThrow(new RuntimeException("Failed!")) + .thenReturn(firstPage).thenReturn(lastPage); + + try { + proxy.getShardList(); + fail("First ListShards call should have failed!"); + } catch (Exception e) { + // Do nothing + } + assertEquals(shards, proxy.getShardList()); + } + + /** + * Tests that if we receive any duplicate shard responses from the service during a shard sync, we dedup the response + * and continue gracefully. + */ + @Test + public void testDuplicateShardResponseDedupedGracefully() { + proxy.setCachedShardMap(null); + List duplicateShards = new ArrayList<>(shards); + duplicateShards.addAll(shards); + ListShardsResult pageOfShards = new ListShardsResult().withShards(duplicateShards).withNextToken(null); + + when(mockClient.listShards(any())).thenReturn(pageOfShards); + + proxy.getShardList(); + assertEquals(shards, proxy.getShardList()); + } + private void mockListShardsForSingleResponse(List shards) { when(mockClient.listShards(any())).thenReturn(listShardsResult); when(listShardsResult.getShards()).thenReturn(shards); From 1956e027253c0c3621099f6aa89131bd29e52896 Mon Sep 17 00:00:00 2001 From: Joshua Kim <20001595+joshua-kim@users.noreply.github.com> Date: Mon, 3 May 2021 15:05:22 -0700 Subject: [PATCH 15/48] Preparing for 1.14.3 release (#812) Co-authored-by: Joshua Kim --- CHANGELOG.md | 6 +++++- README.md | 7 ++++++- pom.xml | 2 +- .../lib/worker/KinesisClientLibConfiguration.java | 2 +- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3e5ce41..eb9435c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,11 @@ # Changelog +### Latest Release (1.14.3 - May 3, 2021) +* [Milestone#60](https://github.com/awslabs/amazon-kinesis-client/milestone/60) +* [#811](https://github.com/awslabs/amazon-kinesis-client/pull/811) Fixing a bug in `KinesisProxy` that can lead to undetermined behavior during partial failures. +* [#811](https://github.com/awslabs/amazon-kinesis-client/pull/811) Adding guardrails to handle duplicate shards from the service. -## Latest Release (1.14.2 - February 24, 2021) +## Release (1.14.2 - February 24, 2021) * [Milestone#57](https://github.com/awslabs/amazon-kinesis-client/milestone/57) * [#790](https://github.com/awslabs/amazon-kinesis-client/pull/790) Fixing a bug that caused paginated `ListShards` calls with the `ShardFilter` parameter to fail when the lease table was being initialized. diff --git a/README.md b/README.md index 4d3ca0fd..49a541eb 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,12 @@ To make it easier for developers to write record processors in other languages, ## Release Notes -## Latest Release (1.14.2 - February 24, 2021) +### Latest Release (1.14.3 - May 3, 2021) +* [Milestone#60](https://github.com/awslabs/amazon-kinesis-client/milestone/60) +* [#811](https://github.com/awslabs/amazon-kinesis-client/pull/811) Fixing a bug in `KinesisProxy` that can lead to undetermined behavior during partial failures. +* [#811](https://github.com/awslabs/amazon-kinesis-client/pull/811) Adding guardrails to handle duplicate shards from the service. + +## Release (1.14.2 - February 24, 2021) * [Milestone#57](https://github.com/awslabs/amazon-kinesis-client/milestone/57) * [#790](https://github.com/awslabs/amazon-kinesis-client/pull/790) Fixing a bug that caused paginated `ListShards` calls with the `ShardFilter` parameter to fail when the lease table was being initialized. diff --git a/pom.xml b/pom.xml index e4064be9..2d4f714d 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ amazon-kinesis-client jar Amazon Kinesis Client Library for Java - 1.14.2 + 1.14.3 The Amazon Kinesis Client Library for Java enables Java developers to easily consume and process data from Amazon Kinesis. diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java index cc385fd2..66f9d1ae 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java @@ -147,7 +147,7 @@ public class KinesisClientLibConfiguration { /** * User agent set when Amazon Kinesis Client Library makes AWS requests. */ - public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.14.2"; + public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.14.3"; /** * KCL will validate client provided sequence numbers with a call to Amazon Kinesis before checkpointing for calls From b314d563085a164c30b54db91fbfa1b9a99b3c6d Mon Sep 17 00:00:00 2001 From: Joshua Kim <20001595+joshua-kim@users.noreply.github.com> Date: Mon, 3 May 2021 18:22:08 -0700 Subject: [PATCH 16/48] Updating to SNAPSHOT version. (#813) Co-authored-by: Joshua Kim --- pom.xml | 2 +- .../clientlibrary/lib/worker/KinesisClientLibConfiguration.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 2d4f714d..12215909 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ amazon-kinesis-client jar Amazon Kinesis Client Library for Java - 1.14.3 + 1.14.4-SNAPSHOT The Amazon Kinesis Client Library for Java enables Java developers to easily consume and process data from Amazon Kinesis. diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java index 66f9d1ae..cecdcaac 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java @@ -147,7 +147,7 @@ public class KinesisClientLibConfiguration { /** * User agent set when Amazon Kinesis Client Library makes AWS requests. */ - public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.14.3"; + public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.14.4-SNAPSHOT"; /** * KCL will validate client provided sequence numbers with a call to Amazon Kinesis before checkpointing for calls From cf71e4e4e457f5fcda77f4e34ae150aa26016f48 Mon Sep 17 00:00:00 2001 From: catalinclapon Date: Thu, 13 May 2021 12:53:52 -0700 Subject: [PATCH 17/48] Update the Worker shutdown logic to make sure that the LeaseCleanupManager also terminates all the threads that it has started (#816) * Update the Worker shutdown logic to make sure that the LeaseCleanupManager also terminates all the threads that it has started * Minor javadoc fixes --- .../clientlibrary/lib/worker/Worker.java | 4 ++++ .../leases/impl/LeaseCleanupManager.java | 17 +++++++++++++++++ .../leases/impl/LeaseCleanupManagerTest.java | 12 ++++++++++++ 3 files changed, 33 insertions(+) diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java index e7e725f8..a69ea6ca 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java @@ -1045,6 +1045,10 @@ public class Worker implements Runnable { // Lost leases will force Worker to begin shutdown process for all shard consumers in // Worker.run(). leaseCoordinator.stop(); + + // Stop the lease cleanup manager + leaseCleanupManager.shutdown(); + // Stop the periodicShardSyncManager for the worker if (shardSyncStrategy != null) { shardSyncStrategy.onWorkerShutDown(); diff --git a/src/main/java/com/amazonaws/services/kinesis/leases/impl/LeaseCleanupManager.java b/src/main/java/com/amazonaws/services/kinesis/leases/impl/LeaseCleanupManager.java index 16f5f353..d19fc3ed 100644 --- a/src/main/java/com/amazonaws/services/kinesis/leases/impl/LeaseCleanupManager.java +++ b/src/main/java/com/amazonaws/services/kinesis/leases/impl/LeaseCleanupManager.java @@ -121,6 +121,23 @@ public class LeaseCleanupManager { } } + /** + * Stops the lease cleanup thread, which is scheduled periodically as specified by + * {@link LeaseCleanupManager#leaseCleanupIntervalMillis} + */ + public void shutdown() { + if (isRunning) { + LOG.info("Stopping the lease cleanup thread."); + completedLeaseStopwatch.stop(); + garbageLeaseStopwatch.stop(); + deletionThreadPool.shutdown(); + + isRunning = false; + } else { + LOG.info("Lease cleanup thread already stopped."); + } + } + /** * Enqueues a lease for deletion without check for duplicate entry. Use {@link #isEnqueuedForDeletion} * for checking the duplicate entries. diff --git a/src/test/java/com/amazonaws/services/kinesis/leases/impl/LeaseCleanupManagerTest.java b/src/test/java/com/amazonaws/services/kinesis/leases/impl/LeaseCleanupManagerTest.java index f89ae644..23fda13d 100644 --- a/src/test/java/com/amazonaws/services/kinesis/leases/impl/LeaseCleanupManagerTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/leases/impl/LeaseCleanupManagerTest.java @@ -93,6 +93,18 @@ public class LeaseCleanupManagerTest { leaseCleanupManager.start(); } + /** + * Tests subsequent calls to shutdown {@link LeaseCleanupManager}. + */ + @Test + public final void testSubsequentShutdowns() { + leaseCleanupManager.start(); + Assert.assertTrue(leaseCleanupManager.isRunning()); + leaseCleanupManager.shutdown(); + Assert.assertFalse(leaseCleanupManager.isRunning()); + leaseCleanupManager.shutdown(); + } + /** * Tests that when both child shard leases are present, we are able to delete the parent shard for the completed * shard case. From fb1f44f482b7bc9c28fee43e33211bdb51b42072 Mon Sep 17 00:00:00 2001 From: Avinash Chowdary Date: Mon, 14 Jun 2021 13:54:29 -0700 Subject: [PATCH 18/48] Upgraded AWS SDK version to 1.12.3 (#821) Co-authored-by: Avinash Ravilla --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 12215909..f15d99ed 100644 --- a/pom.xml +++ b/pom.xml @@ -25,7 +25,7 @@ - 1.11.844 + 1.12.3 1.0.392 libsqlite4java ${project.build.directory}/test-lib From d8e29e8cd788d82b1de2c70a0db4ca74b207b632 Mon Sep 17 00:00:00 2001 From: Avinash Chowdary Date: Mon, 14 Jun 2021 14:36:31 -0700 Subject: [PATCH 19/48] Preparing for 1.14.4 Release (#823) * Upgraded aws java sdk version * preparing for release 1.4.4 Co-authored-by: Avinash Ravilla --- CHANGELOG.md | 7 ++++++- README.md | 7 ++++++- pom.xml | 2 +- .../lib/worker/KinesisClientLibConfiguration.java | 2 +- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb9435c3..c8c85c87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ # Changelog -### Latest Release (1.14.3 - May 3, 2021) +### Latest Release (1.14.4 - June 14, 2021) +* [Milestone#61](https://github.com/awslabs/amazon-kinesis-client/milestone/61) +* [#816](https://github.com/awslabs/amazon-kinesis-client/pull/816) Updated the Worker shutdown logic to make sure that the `LeaseCleanupManager` also terminates all the threads that it has started. +* [#821](https://github.com/awslabs/amazon-kinesis-client/pull/821) Upgrading version of AWS Java SDK to 1.12.3 + +### Release (1.14.3 - May 3, 2021) * [Milestone#60](https://github.com/awslabs/amazon-kinesis-client/milestone/60) * [#811](https://github.com/awslabs/amazon-kinesis-client/pull/811) Fixing a bug in `KinesisProxy` that can lead to undetermined behavior during partial failures. * [#811](https://github.com/awslabs/amazon-kinesis-client/pull/811) Adding guardrails to handle duplicate shards from the service. diff --git a/README.md b/README.md index 49a541eb..59796ceb 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,12 @@ To make it easier for developers to write record processors in other languages, ## Release Notes -### Latest Release (1.14.3 - May 3, 2021) +### Latest Release (1.14.4 - June 14, 2021) +* [Milestone#61](https://github.com/awslabs/amazon-kinesis-client/milestone/61) +* [#816](https://github.com/awslabs/amazon-kinesis-client/pull/816) Updated the Worker shutdown logic to make sure that the `LeaseCleanupManager` also terminates all the threads that it has started. +* [#821](https://github.com/awslabs/amazon-kinesis-client/pull/821) Upgrading version of AWS Java SDK to 1.12.3 + +### Release (1.14.3 - May 3, 2021) * [Milestone#60](https://github.com/awslabs/amazon-kinesis-client/milestone/60) * [#811](https://github.com/awslabs/amazon-kinesis-client/pull/811) Fixing a bug in `KinesisProxy` that can lead to undetermined behavior during partial failures. * [#811](https://github.com/awslabs/amazon-kinesis-client/pull/811) Adding guardrails to handle duplicate shards from the service. diff --git a/pom.xml b/pom.xml index f15d99ed..81a48070 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ amazon-kinesis-client jar Amazon Kinesis Client Library for Java - 1.14.4-SNAPSHOT + 1.14.4 The Amazon Kinesis Client Library for Java enables Java developers to easily consume and process data from Amazon Kinesis. diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java index cecdcaac..ad35f737 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java @@ -147,7 +147,7 @@ public class KinesisClientLibConfiguration { /** * User agent set when Amazon Kinesis Client Library makes AWS requests. */ - public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.14.4-SNAPSHOT"; + public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.14.4"; /** * KCL will validate client provided sequence numbers with a call to Amazon Kinesis before checkpointing for calls From 269724b8df33aa854f4efa776fe8afb8630ead43 Mon Sep 17 00:00:00 2001 From: Rex Chen Date: Fri, 10 Dec 2021 14:27:20 -0500 Subject: [PATCH 20/48] Update log4j dependency Signed-off-by: Rex Chen --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 81a48070..3d86748f 100644 --- a/pom.xml +++ b/pom.xml @@ -104,9 +104,9 @@ - log4j - log4j - 1.2.17 + org.apache.logging.log4j + log4j-core + 2.15.0 test From b402ad69b22a961f7dce1cc7f26b9c1bca728b10 Mon Sep 17 00:00:00 2001 From: Rex Chen Date: Fri, 10 Dec 2021 17:03:40 -0500 Subject: [PATCH 21/48] Upgrade AWSSDK to v1.12.128 Signed-off-by: Rex Chen --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3d86748f..3ed84f23 100644 --- a/pom.xml +++ b/pom.xml @@ -25,7 +25,7 @@ - 1.12.3 + 1.12.128 1.0.392 libsqlite4java ${project.build.directory}/test-lib From f9648b4d990595d33d2a96fa1bf073177e0d0cb5 Mon Sep 17 00:00:00 2001 From: Rex Chen Date: Fri, 10 Dec 2021 16:58:26 -0500 Subject: [PATCH 22/48] Preparation for v1.14.5 Signed-off-by: Rex Chen --- CHANGELOG.md | 6 +++++- README.md | 6 +++++- pom.xml | 2 +- .../lib/worker/KinesisClientLibConfiguration.java | 2 +- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c8c85c87..e74e733a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ # Changelog -### Latest Release (1.14.4 - June 14, 2021) +### Latest Release (1.14.5 - Dec 10, 2021) +* [#872](https://github.com/awslabs/amazon-kinesis-client/pull/872) Update log4j test dependency from 1.2.17 to 2.15.0 +* [#873](https://github.com/awslabs/amazon-kinesis-client/pull/873) Upgrading version of AWS Java SDK to 1.12.128 + +### Release (1.14.4 - June 14, 2021) * [Milestone#61](https://github.com/awslabs/amazon-kinesis-client/milestone/61) * [#816](https://github.com/awslabs/amazon-kinesis-client/pull/816) Updated the Worker shutdown logic to make sure that the `LeaseCleanupManager` also terminates all the threads that it has started. * [#821](https://github.com/awslabs/amazon-kinesis-client/pull/821) Upgrading version of AWS Java SDK to 1.12.3 diff --git a/README.md b/README.md index 59796ceb..9a557adb 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,11 @@ To make it easier for developers to write record processors in other languages, ## Release Notes -### Latest Release (1.14.4 - June 14, 2021) +### Latest Release (1.14.5 - Dec 10, 2021) +* [#872](https://github.com/awslabs/amazon-kinesis-client/pull/872) Update log4j test dependency from 1.2.17 to 2.15.0 +* [#873](https://github.com/awslabs/amazon-kinesis-client/pull/873) Upgrading version of AWS Java SDK to 1.12.128 + +### Release (1.14.4 - June 14, 2021) * [Milestone#61](https://github.com/awslabs/amazon-kinesis-client/milestone/61) * [#816](https://github.com/awslabs/amazon-kinesis-client/pull/816) Updated the Worker shutdown logic to make sure that the `LeaseCleanupManager` also terminates all the threads that it has started. * [#821](https://github.com/awslabs/amazon-kinesis-client/pull/821) Upgrading version of AWS Java SDK to 1.12.3 diff --git a/pom.xml b/pom.xml index 3ed84f23..5430790e 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ amazon-kinesis-client jar Amazon Kinesis Client Library for Java - 1.14.4 + 1.14.5 The Amazon Kinesis Client Library for Java enables Java developers to easily consume and process data from Amazon Kinesis. diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java index ad35f737..ca345c5d 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java @@ -147,7 +147,7 @@ public class KinesisClientLibConfiguration { /** * User agent set when Amazon Kinesis Client Library makes AWS requests. */ - public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.14.4"; + public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.14.5"; /** * KCL will validate client provided sequence numbers with a call to Amazon Kinesis before checkpointing for calls From 07c86081fbfdb8937fb78c0ffc4b7593f4788891 Mon Sep 17 00:00:00 2001 From: Rex Chen Date: Wed, 15 Dec 2021 12:47:58 -0500 Subject: [PATCH 23/48] Upgrade log4j test dependency to v2.16.0 Signed-off-by: Rex Chen --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5430790e..486b4a29 100644 --- a/pom.xml +++ b/pom.xml @@ -106,7 +106,7 @@ org.apache.logging.log4j log4j-core - 2.15.0 + 2.16.0 test From 58036d4977a4b69949556447e072e40da73a2f4d Mon Sep 17 00:00:00 2001 From: Rex Chen Date: Wed, 15 Dec 2021 13:07:16 -0500 Subject: [PATCH 24/48] Preparation for v1.14.6 Signed-off-by: Rex Chen --- CHANGELOG.md | 5 ++++- README.md | 5 ++++- pom.xml | 2 +- .../lib/worker/KinesisClientLibConfiguration.java | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e74e733a..65bc1cce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ # Changelog -### Latest Release (1.14.5 - Dec 10, 2021) +### Latest Release (1.14.6 - Dec 15, 2021) +* [#876](https://github.com/awslabs/amazon-kinesis-client/pull/876) Update log4j test dependency from 2.15.0 to 2.16.0 + +### Release (1.14.5 - Dec 10, 2021) * [#872](https://github.com/awslabs/amazon-kinesis-client/pull/872) Update log4j test dependency from 1.2.17 to 2.15.0 * [#873](https://github.com/awslabs/amazon-kinesis-client/pull/873) Upgrading version of AWS Java SDK to 1.12.128 diff --git a/README.md b/README.md index 9a557adb..dab65cf6 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,10 @@ To make it easier for developers to write record processors in other languages, ## Release Notes -### Latest Release (1.14.5 - Dec 10, 2021) +### Latest Release (1.14.6 - Dec 15, 2021) +* [#876](https://github.com/awslabs/amazon-kinesis-client/pull/876) Update log4j test dependency from 2.15.0 to 2.16.0 + +### Release (1.14.5 - Dec 10, 2021) * [#872](https://github.com/awslabs/amazon-kinesis-client/pull/872) Update log4j test dependency from 1.2.17 to 2.15.0 * [#873](https://github.com/awslabs/amazon-kinesis-client/pull/873) Upgrading version of AWS Java SDK to 1.12.128 diff --git a/pom.xml b/pom.xml index 486b4a29..f12c2f36 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ amazon-kinesis-client jar Amazon Kinesis Client Library for Java - 1.14.5 + 1.14.6 The Amazon Kinesis Client Library for Java enables Java developers to easily consume and process data from Amazon Kinesis. diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java index ca345c5d..9a85df8e 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java @@ -147,7 +147,7 @@ public class KinesisClientLibConfiguration { /** * User agent set when Amazon Kinesis Client Library makes AWS requests. */ - public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.14.5"; + public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.14.6"; /** * KCL will validate client provided sequence numbers with a call to Amazon Kinesis before checkpointing for calls From 1ead07cf9ff25b47f8e6c87ac496298d631c5e83 Mon Sep 17 00:00:00 2001 From: Yu Zeng Date: Tue, 21 Dec 2021 16:02:15 -0800 Subject: [PATCH 25/48] Update log4j to 2.17.0 and couple of other dependencies (#881) * Configure dependabot * Bump guava from 26.0-jre to 31.0.1-jre (#26) Bumps [guava](https://github.com/google/guava) from 26.0-jre to 31.0.1-jre. - [Release notes](https://github.com/google/guava/releases) - [Commits](https://github.com/google/guava/commits) --- updated-dependencies: - dependency-name: com.google.guava:guava dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump junit from 4.11 to 4.13.2 (#28) Bumps [junit](https://github.com/junit-team/junit4) from 4.11 to 4.13.2. - [Release notes](https://github.com/junit-team/junit4/releases) - [Changelog](https://github.com/junit-team/junit4/blob/main/doc/ReleaseNotes4.11.md) - [Commits](https://github.com/junit-team/junit4/compare/r4.11...r4.13.2) --- updated-dependencies: - dependency-name: junit:junit dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump aws-java-sdk.version from 1.12.128 to 1.12.130 (#27) Bumps `aws-java-sdk.version` from 1.12.128 to 1.12.130. Updates `aws-java-sdk-dynamodb` from 1.12.128 to 1.12.130 - [Release notes](https://github.com/aws/aws-sdk-java/releases) - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.12.128...1.12.130) Updates `aws-java-sdk-kinesis` from 1.12.128 to 1.12.130 - [Release notes](https://github.com/aws/aws-sdk-java/releases) - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.12.128...1.12.130) Updates `aws-java-sdk-cloudwatch` from 1.12.128 to 1.12.130 - [Release notes](https://github.com/aws/aws-sdk-java/releases) - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.12.128...1.12.130) --- updated-dependencies: - dependency-name: com.amazonaws:aws-java-sdk-dynamodb dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: com.amazonaws:aws-java-sdk-kinesis dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: com.amazonaws:aws-java-sdk-cloudwatch dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump protobuf-java from 3.11.4 to 3.19.1 (#25) Bumps [protobuf-java](https://github.com/protocolbuffers/protobuf) from 3.11.4 to 3.19.1. - [Release notes](https://github.com/protocolbuffers/protobuf/releases) - [Changelog](https://github.com/protocolbuffers/protobuf/blob/master/generate_changelog.py) - [Commits](https://github.com/protocolbuffers/protobuf/compare/v3.11.4...v3.19.1) --- updated-dependencies: - dependency-name: com.google.protobuf:protobuf-java dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump maven-failsafe-plugin from 2.19.1 to 2.22.2 (#23) Bumps [maven-failsafe-plugin](https://github.com/apache/maven-surefire) from 2.19.1 to 2.22.2. - [Release notes](https://github.com/apache/maven-surefire/releases) - [Commits](https://github.com/apache/maven-surefire/compare/surefire-2.19.1...surefire-2.22.2) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-failsafe-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Remove dependabot configuration in v1.x branch as it's been configured in master * Bump maven-gpg-plugin from 1.6 to 3.0.1 (#33) Bumps [maven-gpg-plugin](https://github.com/apache/maven-gpg-plugin) from 1.6 to 3.0.1. - [Release notes](https://github.com/apache/maven-gpg-plugin/releases) - [Commits](https://github.com/apache/maven-gpg-plugin/compare/maven-gpg-plugin-1.6...maven-gpg-plugin-3.0.1) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-gpg-plugin dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump maven-compiler-plugin from 3.2 to 3.8.1 (#32) Bumps [maven-compiler-plugin](https://github.com/apache/maven-compiler-plugin) from 3.2 to 3.8.1. - [Release notes](https://github.com/apache/maven-compiler-plugin/releases) - [Commits](https://github.com/apache/maven-compiler-plugin/compare/maven-compiler-plugin-3.2...maven-compiler-plugin-3.8.1) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-compiler-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump lombok from 1.16.10 to 1.18.22 (#31) Bumps [lombok](https://github.com/projectlombok/lombok) from 1.16.10 to 1.18.22. - [Release notes](https://github.com/projectlombok/lombok/releases) - [Changelog](https://github.com/projectlombok/lombok/blob/master/doc/changelog.markdown) - [Commits](https://github.com/projectlombok/lombok/compare/v1.16.10...v1.18.22) --- updated-dependencies: - dependency-name: org.projectlombok:lombok dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump maven-javadoc-plugin from 2.10.3 to 3.3.1 (#30) Bumps [maven-javadoc-plugin](https://github.com/apache/maven-javadoc-plugin) from 2.10.3 to 3.3.1. - [Release notes](https://github.com/apache/maven-javadoc-plugin/releases) - [Commits](https://github.com/apache/maven-javadoc-plugin/compare/maven-javadoc-plugin-2.10.3...maven-javadoc-plugin-3.3.1) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-javadoc-plugin dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump maven-surefire-plugin from 2.19.1 to 2.22.2 (#29) Bumps [maven-surefire-plugin](https://github.com/apache/maven-surefire) from 2.19.1 to 2.22.2. - [Release notes](https://github.com/apache/maven-surefire/releases) - [Commits](https://github.com/apache/maven-surefire/compare/surefire-2.19.1...surefire-2.22.2) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-surefire-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump log4j-core from 2.16.0 to 2.17.0 (#36) Bumps log4j-core from 2.16.0 to 2.17.0. --- updated-dependencies: - dependency-name: org.apache.logging.log4j:log4j-core dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump commons-lang3 from 3.7 to 3.12.0 (#38) Bumps commons-lang3 from 3.7 to 3.12.0. --- updated-dependencies: - dependency-name: org.apache.commons:commons-lang3 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump DynamoDBLocal from 1.11.86 to 1.17.2 (#37) Bumps DynamoDBLocal from 1.11.86 to 1.17.2. --- updated-dependencies: - dependency-name: com.amazonaws:DynamoDBLocal dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump maven-source-plugin from 3.0.1 to 3.2.1 (#35) Bumps [maven-source-plugin](https://github.com/apache/maven-source-plugin) from 3.0.1 to 3.2.1. - [Release notes](https://github.com/apache/maven-source-plugin/releases) - [Commits](https://github.com/apache/maven-source-plugin/compare/maven-source-plugin-3.0.1...maven-source-plugin-3.2.1) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-source-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump commons-logging from 1.1.3 to 1.2 (#34) Bumps commons-logging from 1.1.3 to 1.2. --- updated-dependencies: - dependency-name: commons-logging:commons-logging dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Add test dependencies Co-authored-by: Yu Zeng Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 43 +++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/pom.xml b/pom.xml index f12c2f36..dc31f6d6 100644 --- a/pom.xml +++ b/pom.xml @@ -25,13 +25,18 @@ - 1.12.128 + 1.12.130 1.0.392 libsqlite4java ${project.build.directory}/test-lib + + com.amazonaws + aws-java-sdk-core + ${aws-java-sdk.version} + com.amazonaws aws-java-sdk-dynamodb @@ -50,27 +55,27 @@ com.google.guava guava - 26.0-jre + 31.0.1-jre com.google.protobuf protobuf-java - 3.11.4 + 3.19.1 org.apache.commons commons-lang3 - 3.7 + 3.12.0 commons-logging commons-logging - 1.1.3 + 1.2 org.projectlombok lombok - 1.16.10 + 1.18.22 provided @@ -78,7 +83,7 @@ junit junit - 4.11 + 4.13.2 test @@ -99,16 +104,22 @@ com.amazonaws DynamoDBLocal - 1.11.86 + 1.17.2 test org.apache.logging.log4j log4j-core - 2.16.0 + 2.17.0 test + + + com.fasterxml.jackson.core + jackson-core + 2.13.0 + @@ -136,7 +147,7 @@ org.apache.maven.plugins maven-compiler-plugin - 3.2 + 3.8.1 1.8 1.8 @@ -150,7 +161,7 @@ org.apache.maven.plugins maven-surefire-plugin - 2.19.1 + 2.22.2 **/*IntegrationTest.java @@ -166,7 +177,7 @@ org.apache.maven.plugins maven-failsafe-plugin - 2.19.1 + 2.22.2 **/*IntegrationTest.java @@ -252,7 +263,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 2.10.3 + 3.3.1 com.amazonaws.services.kinesis.producer.protobuf @@ -268,7 +279,7 @@ org.apache.maven.plugins maven-source-plugin - 3.0.1 + 3.2.1 attach-sources @@ -300,7 +311,7 @@ [1.8,) - -Xdoclint:none + none @@ -311,7 +322,7 @@ org.apache.maven.plugins maven-gpg-plugin - 1.6 + 3.0.1 sign-artifacts From 4fd7b7aadb6a0c866fef4ac9030f09aa97faf45b Mon Sep 17 00:00:00 2001 From: Yu Zeng Date: Wed, 22 Dec 2021 10:47:58 -0800 Subject: [PATCH 26/48] Preparation for v1.14.7 (#884) Co-authored-by: Yu Zeng --- CHANGELOG.md | 5 ++++- README.md | 5 ++++- pom.xml | 2 +- .../lib/worker/KinesisClientLibConfiguration.java | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65bc1cce..72bfe68b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ # Changelog -### Latest Release (1.14.6 - Dec 15, 2021) +### Latest Release (1.14.7 - Dec 22, 2021) +* [#881](https://github.com/awslabs/amazon-kinesis-client/pull/881) Update log4j test dependency from 2.16.0 to 2.17.0 and some other dependencies + +### Release (1.14.6 - Dec 15, 2021) * [#876](https://github.com/awslabs/amazon-kinesis-client/pull/876) Update log4j test dependency from 2.15.0 to 2.16.0 ### Release (1.14.5 - Dec 10, 2021) diff --git a/README.md b/README.md index dab65cf6..c0baa3db 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,10 @@ To make it easier for developers to write record processors in other languages, ## Release Notes -### Latest Release (1.14.6 - Dec 15, 2021) +### Latest Release (1.14.7 - Dec 22, 2021) +* [#881](https://github.com/awslabs/amazon-kinesis-client/pull/881) Update log4j test dependency from 2.16.0 to 2.17.0 and some other dependencies + +### Release (1.14.6 - Dec 15, 2021) * [#876](https://github.com/awslabs/amazon-kinesis-client/pull/876) Update log4j test dependency from 2.15.0 to 2.16.0 ### Release (1.14.5 - Dec 10, 2021) diff --git a/pom.xml b/pom.xml index dc31f6d6..9a45f8a5 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ amazon-kinesis-client jar Amazon Kinesis Client Library for Java - 1.14.6 + 1.14.7 The Amazon Kinesis Client Library for Java enables Java developers to easily consume and process data from Amazon Kinesis. diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java index 9a85df8e..b603fdfc 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java @@ -147,7 +147,7 @@ public class KinesisClientLibConfiguration { /** * User agent set when Amazon Kinesis Client Library makes AWS requests. */ - public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.14.6"; + public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.14.7"; /** * KCL will validate client provided sequence numbers with a call to Amazon Kinesis before checkpointing for calls From 37b5d7b9a1ccad483469ef542a6a7237462b14f2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Feb 2022 13:20:18 +0000 Subject: [PATCH 27/48] Bump maven-compiler-plugin from 3.8.1 to 3.10.0 Bumps [maven-compiler-plugin](https://github.com/apache/maven-compiler-plugin) from 3.8.1 to 3.10.0. - [Release notes](https://github.com/apache/maven-compiler-plugin/releases) - [Commits](https://github.com/apache/maven-compiler-plugin/compare/maven-compiler-plugin-3.8.1...maven-compiler-plugin-3.10.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-compiler-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9a45f8a5..18f840cb 100644 --- a/pom.xml +++ b/pom.xml @@ -147,7 +147,7 @@ org.apache.maven.plugins maven-compiler-plugin - 3.8.1 + 3.10.0 1.8 1.8 From a809b12c43c57a3d6ad3827feb60e4322614259c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Jan 2022 13:19:22 +0000 Subject: [PATCH 28/48] Bump protobuf-java from 3.19.1 to 3.19.4 Bumps [protobuf-java](https://github.com/protocolbuffers/protobuf) from 3.19.1 to 3.19.4. - [Release notes](https://github.com/protocolbuffers/protobuf/releases) - [Changelog](https://github.com/protocolbuffers/protobuf/blob/master/generate_changelog.py) - [Commits](https://github.com/protocolbuffers/protobuf/compare/v3.19.1...v3.19.4) --- updated-dependencies: - dependency-name: com.google.protobuf:protobuf-java dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 18f840cb..0644943e 100644 --- a/pom.xml +++ b/pom.xml @@ -60,7 +60,7 @@ com.google.protobuf protobuf-java - 3.19.1 + 3.19.4 org.apache.commons From 94b138a9d9a502ee0f4f000bb0efd2766ebadc37 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 Dec 2021 19:51:56 +0000 Subject: [PATCH 29/48] Bump log4j-core from 2.17.0 to 2.17.1 Bumps log4j-core from 2.17.0 to 2.17.1. --- updated-dependencies: - dependency-name: org.apache.logging.log4j:log4j-core dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0644943e..309392b8 100644 --- a/pom.xml +++ b/pom.xml @@ -111,7 +111,7 @@ org.apache.logging.log4j log4j-core - 2.17.0 + 2.17.1 test From 8dceb3f80f4015481263787a6179187c6765e4fd Mon Sep 17 00:00:00 2001 From: Yu Zeng Date: Thu, 24 Feb 2022 14:17:00 -0800 Subject: [PATCH 30/48] Preparing for release 1.4.8 --- CHANGELOG.md | 7 ++++++- README.md | 7 ++++++- pom.xml | 2 +- .../lib/worker/KinesisClientLibConfiguration.java | 2 +- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72bfe68b..43772079 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ # Changelog -### Latest Release (1.14.7 - Dec 22, 2021) +### Latest Release (1.14.8 - Feb 24, 2021) +* [Bump log4j-core from 2.17.0 to 2.17.1](https://github.com/awslabs/amazon-kinesis-client/commit/94b138a9d9a502ee0f4f000bb0efd2766ebadc37) +* [Bump protobuf-java from 3.19.1 to 3.19.4](https://github.com/awslabs/amazon-kinesis-client/commit/a809b12c43c57a3d6ad3827feb60e4322614259c) +* [Bump maven-compiler-plugin from 3.8.1 to 3.10.0](https://github.com/awslabs/amazon-kinesis-client/commit/37b5d7b9a1ccad483469ef542a6a7237462b14f2) + +### Release (1.14.7 - Dec 22, 2021) * [#881](https://github.com/awslabs/amazon-kinesis-client/pull/881) Update log4j test dependency from 2.16.0 to 2.17.0 and some other dependencies ### Release (1.14.6 - Dec 15, 2021) diff --git a/README.md b/README.md index c0baa3db..5da7a029 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,12 @@ To make it easier for developers to write record processors in other languages, ## Release Notes -### Latest Release (1.14.7 - Dec 22, 2021) +### Latest Release (1.14.8 - Feb 24, 2021) +* [Bump log4j-core from 2.17.0 to 2.17.1](https://github.com/awslabs/amazon-kinesis-client/commit/94b138a9d9a502ee0f4f000bb0efd2766ebadc37) +* [Bump protobuf-java from 3.19.1 to 3.19.4](https://github.com/awslabs/amazon-kinesis-client/commit/a809b12c43c57a3d6ad3827feb60e4322614259c) +* [Bump maven-compiler-plugin from 3.8.1 to 3.10.0](https://github.com/awslabs/amazon-kinesis-client/commit/37b5d7b9a1ccad483469ef542a6a7237462b14f2) + +### Release (1.14.7 - Dec 22, 2021) * [#881](https://github.com/awslabs/amazon-kinesis-client/pull/881) Update log4j test dependency from 2.16.0 to 2.17.0 and some other dependencies ### Release (1.14.6 - Dec 15, 2021) diff --git a/pom.xml b/pom.xml index 309392b8..e87ede7d 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ amazon-kinesis-client jar Amazon Kinesis Client Library for Java - 1.14.7 + 1.14.8 The Amazon Kinesis Client Library for Java enables Java developers to easily consume and process data from Amazon Kinesis. diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java index b603fdfc..ccde83f3 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java @@ -147,7 +147,7 @@ public class KinesisClientLibConfiguration { /** * User agent set when Amazon Kinesis Client Library makes AWS requests. */ - public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.14.7"; + public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.14.8"; /** * KCL will validate client provided sequence numbers with a call to Amazon Kinesis before checkpointing for calls From 251b331a2e0fd912b50f8b5a12d088bf0b3263b9 Mon Sep 17 00:00:00 2001 From: Nicholas Gutierrez <108950398+nichgu@users.noreply.github.com> Date: Thu, 8 Sep 2022 17:45:53 -0700 Subject: [PATCH 31/48] PeriodicShardSyncManager Changes Needed for DynamoDBStreamsKinesisAdapter Compatibility (#970) * Interface and Rename for PeriodicShardSyncManager * Removed Automatic Indents * More Auto Indent Fixes --- .../lib/worker/IPeriodicShardSyncManager.java | 28 ++ .../clientlibrary/lib/worker/ITask.java | 2 +- .../KinesisPeriodicShardSyncManager.java | 403 ++++++++++++++++++ .../MetricsCollectingTaskDecorator.java | 2 +- .../lib/worker/PeriodicShardSyncStrategy.java | 4 +- .../lib/worker/ShardEndShardSyncStrategy.java | 4 +- .../lib/worker/ShardSyncTask.java | 4 +- .../clientlibrary/lib/worker/TaskResult.java | 6 +- .../clientlibrary/lib/worker/Worker.java | 26 +- 9 files changed, 459 insertions(+), 20 deletions(-) create mode 100644 src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/IPeriodicShardSyncManager.java create mode 100644 src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisPeriodicShardSyncManager.java diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/IPeriodicShardSyncManager.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/IPeriodicShardSyncManager.java new file mode 100644 index 00000000..15bc007a --- /dev/null +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/IPeriodicShardSyncManager.java @@ -0,0 +1,28 @@ +package com.amazonaws.services.kinesis.clientlibrary.lib.worker; + +import com.google.common.annotations.VisibleForTesting; +import lombok.Value; +import lombok.experimental.Accessors; + +public interface IPeriodicShardSyncManager { + + TaskResult start(); + + /** + * Runs ShardSync once, without scheduling further periodic ShardSyncs. + * @return TaskResult from shard sync + */ + TaskResult syncShardsOnce(); + + void stop(); + + @Value + @Accessors(fluent = true) + @VisibleForTesting + class ShardSyncResponse { + private final boolean shouldDoShardSync; + private final boolean isHoleDetected; + private final String reasonForDecision; + } +} + diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ITask.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ITask.java index fc2000a0..10c02b6e 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ITask.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ITask.java @@ -20,7 +20,7 @@ import java.util.concurrent.Callable; * Interface for shard processing tasks. * A task may execute an application callback (e.g. initialize, process, shutdown). */ -interface ITask extends Callable { +public interface ITask extends Callable { /** * Perform task logic. diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisPeriodicShardSyncManager.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisPeriodicShardSyncManager.java new file mode 100644 index 00000000..99127291 --- /dev/null +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisPeriodicShardSyncManager.java @@ -0,0 +1,403 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.amazonaws.services.kinesis.clientlibrary.lib.worker; + +import java.io.Serializable; +import java.math.BigInteger; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import com.amazonaws.services.cloudwatch.model.StandardUnit; +import com.amazonaws.services.kinesis.clientlibrary.proxies.IKinesisProxy; +import com.amazonaws.services.kinesis.leases.exceptions.DependencyException; +import com.amazonaws.services.kinesis.leases.exceptions.InvalidStateException; +import com.amazonaws.services.kinesis.leases.exceptions.ProvisionedThroughputException; +import com.amazonaws.services.kinesis.leases.impl.HashKeyRangeForLease; +import com.amazonaws.services.kinesis.leases.impl.KinesisClientLease; +import com.amazonaws.services.kinesis.leases.impl.UpdateField; +import com.amazonaws.services.kinesis.leases.interfaces.ILeaseManager; +import com.amazonaws.services.kinesis.metrics.impl.MetricsHelper; +import com.amazonaws.services.kinesis.metrics.interfaces.IMetricsFactory; +import com.amazonaws.services.kinesis.metrics.interfaces.MetricsLevel; +import com.amazonaws.services.kinesis.model.Shard; +import com.amazonaws.util.CollectionUtils; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ComparisonChain; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NonNull; +import lombok.Value; +import org.apache.commons.lang3.Validate; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import static com.amazonaws.services.kinesis.leases.impl.HashKeyRangeForLease.fromHashKeyRange; + +/** + * The top level orchestrator for coordinating the periodic shard sync related activities. If the configured + * {@link ShardSyncStrategyType} is PERIODIC, this class will be the main shard sync orchestrator. For non-PERIODIC + * strategies, this class will serve as an internal auditor that periodically checks if the full hash range is covered + * by currently held leases, and initiates a recovery shard sync if not. + */ +@Getter +@EqualsAndHashCode +class KinesisPeriodicShardSyncManager implements IPeriodicShardSyncManager{ + private static final Log LOG = LogFactory.getLog(KinesisPeriodicShardSyncManager.class); + private static final long INITIAL_DELAY = 0; + + /** DEFAULT interval is used for PERIODIC {@link ShardSyncStrategyType}. */ + + private static final long DEFAULT_PERIODIC_SHARD_SYNC_INTERVAL_MILLIS = 1000L; + + /** Parameters for validating hash range completeness when running in auditor mode. */ + @VisibleForTesting + static final BigInteger MIN_HASH_KEY = BigInteger.ZERO; + @VisibleForTesting + static final BigInteger MAX_HASH_KEY = new BigInteger("2").pow(128).subtract(BigInteger.ONE); + static final String PERIODIC_SHARD_SYNC_MANAGER = "PeriodicShardSyncManager"; + private final HashRangeHoleTracker hashRangeHoleTracker = new HashRangeHoleTracker(); + + private final String workerId; + private final LeaderDecider leaderDecider; + private final ITask metricsEmittingShardSyncTask; + private final ScheduledExecutorService shardSyncThreadPool; + private final ILeaseManager leaseManager; + private final IKinesisProxy kinesisProxy; + private final boolean isAuditorMode; + private final long periodicShardSyncIntervalMillis; + private boolean isRunning; + private final IMetricsFactory metricsFactory; + private final int leasesRecoveryAuditorInconsistencyConfidenceThreshold; + + + KinesisPeriodicShardSyncManager(String workerId, + LeaderDecider leaderDecider, + ShardSyncTask shardSyncTask, + IMetricsFactory metricsFactory, + ILeaseManager leaseManager, + IKinesisProxy kinesisProxy, + boolean isAuditorMode, + long leasesRecoveryAuditorExecutionFrequencyMillis, + int leasesRecoveryAuditorInconsistencyConfidenceThreshold) { + this(workerId, leaderDecider, shardSyncTask, Executors.newSingleThreadScheduledExecutor(), metricsFactory, + leaseManager, kinesisProxy, isAuditorMode, leasesRecoveryAuditorExecutionFrequencyMillis, + leasesRecoveryAuditorInconsistencyConfidenceThreshold); + } + + KinesisPeriodicShardSyncManager(String workerId, + LeaderDecider leaderDecider, + ShardSyncTask shardSyncTask, + ScheduledExecutorService shardSyncThreadPool, + IMetricsFactory metricsFactory, + ILeaseManager leaseManager, + IKinesisProxy kinesisProxy, + boolean isAuditorMode, + long leasesRecoveryAuditorExecutionFrequencyMillis, + int leasesRecoveryAuditorInconsistencyConfidenceThreshold) { + Validate.notBlank(workerId, "WorkerID is required to initialize PeriodicShardSyncManager."); + Validate.notNull(leaderDecider, "LeaderDecider is required to initialize PeriodicShardSyncManager."); + Validate.notNull(shardSyncTask, "ShardSyncTask is required to initialize PeriodicShardSyncManager."); + this.workerId = workerId; + this.leaderDecider = leaderDecider; + this.metricsEmittingShardSyncTask = new MetricsCollectingTaskDecorator(shardSyncTask, metricsFactory); + this.shardSyncThreadPool = shardSyncThreadPool; + this.leaseManager = leaseManager; + this.kinesisProxy = kinesisProxy; + this.metricsFactory = metricsFactory; + this.isAuditorMode = isAuditorMode; + this.leasesRecoveryAuditorInconsistencyConfidenceThreshold = leasesRecoveryAuditorInconsistencyConfidenceThreshold; + if (isAuditorMode) { + Validate.notNull(this.leaseManager, "LeaseManager is required for non-PERIODIC shard sync strategies."); + Validate.notNull(this.kinesisProxy, "KinesisProxy is required for non-PERIODIC shard sync strategies."); + this.periodicShardSyncIntervalMillis = leasesRecoveryAuditorExecutionFrequencyMillis; + } else { + this.periodicShardSyncIntervalMillis = DEFAULT_PERIODIC_SHARD_SYNC_INTERVAL_MILLIS; + } + } + + @Override + public synchronized TaskResult start() { + if (!isRunning) { + final Runnable periodicShardSyncer = () -> { + try { + runShardSync(); + } catch (Throwable t) { + LOG.error("Error running shard sync.", t); + } + }; + + shardSyncThreadPool + .scheduleWithFixedDelay(periodicShardSyncer, INITIAL_DELAY, periodicShardSyncIntervalMillis, + TimeUnit.MILLISECONDS); + isRunning = true; + } + return new TaskResult(null); + } + + /** + * Runs ShardSync once, without scheduling further periodic ShardSyncs. + * @return TaskResult from shard sync + */ + @Override + public synchronized TaskResult syncShardsOnce() { + LOG.info("Syncing shards once from worker " + workerId); + return metricsEmittingShardSyncTask.call(); + } + + @Override + public void stop() { + if (isRunning) { + LOG.info(String.format("Shutting down leader decider on worker %s", workerId)); + leaderDecider.shutdown(); + LOG.info(String.format("Shutting down periodic shard sync task scheduler on worker %s", workerId)); + shardSyncThreadPool.shutdown(); + isRunning = false; + } + } + + private void runShardSync() { + if (leaderDecider.isLeader(workerId)) { + LOG.debug("WorkerId " + workerId + " is a leader, running the shard sync task"); + + MetricsHelper.startScope(metricsFactory, PERIODIC_SHARD_SYNC_MANAGER); + boolean isRunSuccess = false; + final long runStartMillis = System.currentTimeMillis(); + + try { + final ShardSyncResponse shardSyncResponse = checkForShardSync(); + MetricsHelper.getMetricsScope().addData("NumStreamsToSync", shardSyncResponse.shouldDoShardSync() ? 1 : 0, StandardUnit.Count, MetricsLevel.SUMMARY); + MetricsHelper.getMetricsScope().addData("NumStreamsWithPartialLeases", shardSyncResponse.isHoleDetected() ? 1 : 0, StandardUnit.Count, MetricsLevel.SUMMARY); + if (shardSyncResponse.shouldDoShardSync()) { + LOG.info("Periodic shard syncer initiating shard sync due to the reason - " + + shardSyncResponse.reasonForDecision()); + metricsEmittingShardSyncTask.call(); + } else { + LOG.info("Skipping shard sync due to the reason - " + shardSyncResponse.reasonForDecision()); + } + isRunSuccess = true; + } catch (Exception e) { + LOG.error("Caught exception while running periodic shard syncer.", e); + } finally { + MetricsHelper.addSuccessAndLatency(runStartMillis, isRunSuccess, MetricsLevel.SUMMARY); + MetricsHelper.endScope(); + } + } else { + LOG.debug("WorkerId " + workerId + " is not a leader, not running the shard sync task"); + } + } + + @VisibleForTesting + ShardSyncResponse checkForShardSync() throws DependencyException, InvalidStateException, + ProvisionedThroughputException { + + if (!isAuditorMode) { + // If we are running with PERIODIC shard sync strategy, we should sync every time. + return new ShardSyncResponse(true, false, "Syncing every time with PERIODIC shard sync strategy."); + } + + // Get current leases from DynamoDB. + final List currentLeases = leaseManager.listLeases(); + + if (CollectionUtils.isNullOrEmpty(currentLeases)) { + // If the current leases are null or empty, then we need to initiate a shard sync. + LOG.info("No leases found. Will trigger a shard sync."); + return new ShardSyncResponse(true, false, "No leases found."); + } + + // Check if there are any holes in the hash range covered by current leases. Return the first hole if present. + Optional hashRangeHoleOpt = hasHoleInLeases(currentLeases); + if (hashRangeHoleOpt.isPresent()) { + // If hole is present, check if the hole is detected consecutively in previous occurrences. If hole is + // determined with high confidence, return true; return false otherwise. We use the high confidence factor + // to avoid shard sync on any holes during resharding and lease cleanups, or other intermittent issues. + final boolean hasHoleWithHighConfidence = + hashRangeHoleTracker.hashHighConfidenceOfHoleWith(hashRangeHoleOpt.get()); + + return new ShardSyncResponse(hasHoleWithHighConfidence, true, + "Detected the same hole for " + hashRangeHoleTracker.getNumConsecutiveHoles() + " times. " + + "Will initiate shard sync after reaching threshold: " + leasesRecoveryAuditorInconsistencyConfidenceThreshold); + } else { + // If hole is not present, clear any previous hole tracking and return false. + hashRangeHoleTracker.reset(); + return new ShardSyncResponse(false, false, "Hash range is complete."); + } + } + + @VisibleForTesting + Optional hasHoleInLeases(List leases) { + // Filter out any leases with checkpoints other than SHARD_END + final List activeLeases = leases.stream() + .filter(lease -> lease.getCheckpoint() != null && !lease.getCheckpoint().isShardEnd()) + .collect(Collectors.toList()); + + final List activeLeasesWithHashRanges = fillWithHashRangesIfRequired(activeLeases); + return checkForHoleInHashKeyRanges(activeLeasesWithHashRanges); + } + + private List fillWithHashRangesIfRequired(List activeLeases) { + final List activeLeasesWithNoHashRanges = activeLeases.stream() + .filter(lease -> lease.getHashKeyRange() == null).collect(Collectors.toList()); + + if (activeLeasesWithNoHashRanges.isEmpty()) { + return activeLeases; + } + + // Fetch shards from Kinesis to fill in the in-memory hash ranges + final Map kinesisShards = kinesisProxy.getShardList().stream() + .collect(Collectors.toMap(Shard::getShardId, shard -> shard)); + + return activeLeases.stream().map(lease -> { + if (lease.getHashKeyRange() == null) { + final String shardId = lease.getLeaseKey(); + final Shard shard = kinesisShards.get(shardId); + if (shard == null) { + return lease; + } + lease.setHashKeyRange(fromHashKeyRange(shard.getHashKeyRange())); + + try { + leaseManager.updateLeaseWithMetaInfo(lease, UpdateField.HASH_KEY_RANGE); + } catch (Exception e) { + LOG.warn("Unable to update hash range information for lease " + lease.getLeaseKey() + + ". This may result in explicit lease sync."); + } + } + return lease; + }).filter(lease -> lease.getHashKeyRange() != null).collect(Collectors.toList()); + } + + @VisibleForTesting + static Optional checkForHoleInHashKeyRanges(List leasesWithHashKeyRanges) { + // Sort the hash ranges by starting hash key + final List sortedLeasesWithHashKeyRanges = sortLeasesByHashRange(leasesWithHashKeyRanges); + if (sortedLeasesWithHashKeyRanges.isEmpty()) { + LOG.error("No leases with valid hash ranges found."); + return Optional.of(new HashRangeHole()); + } + + // Validate the hash range bounds + final KinesisClientLease minHashKeyLease = sortedLeasesWithHashKeyRanges.get(0); + final KinesisClientLease maxHashKeyLease = + sortedLeasesWithHashKeyRanges.get(sortedLeasesWithHashKeyRanges.size() - 1); + if (!minHashKeyLease.getHashKeyRange().startingHashKey().equals(MIN_HASH_KEY) || + !maxHashKeyLease.getHashKeyRange().endingHashKey().equals(MAX_HASH_KEY)) { + LOG.error("Incomplete hash range found between " + minHashKeyLease + " and " + maxHashKeyLease); + return Optional.of(new HashRangeHole(minHashKeyLease.getHashKeyRange(), maxHashKeyLease.getHashKeyRange())); + } + + // Check for any holes in the sorted hash range intervals + if (sortedLeasesWithHashKeyRanges.size() > 1) { + KinesisClientLease leftmostLeaseToReportInCaseOfHole = minHashKeyLease; + HashKeyRangeForLease leftLeaseHashRange = leftmostLeaseToReportInCaseOfHole.getHashKeyRange(); + + for (int i = 1; i < sortedLeasesWithHashKeyRanges.size(); i++) { + final KinesisClientLease rightLease = sortedLeasesWithHashKeyRanges.get(i); + final HashKeyRangeForLease rightLeaseHashRange = rightLease.getHashKeyRange(); + final BigInteger rangeDiff = + rightLeaseHashRange.startingHashKey().subtract(leftLeaseHashRange.endingHashKey()); + // We have overlapping leases when rangeDiff is 0 or negative. + // signum() will be -1 for negative and 0 if value is 0. + // Merge the ranges for further tracking. + if (rangeDiff.signum() <= 0) { + leftLeaseHashRange = new HashKeyRangeForLease(leftLeaseHashRange.startingHashKey(), + leftLeaseHashRange.endingHashKey().max(rightLeaseHashRange.endingHashKey())); + } else { + // We have non-overlapping leases when rangeDiff is positive. signum() will be 1 in this case. + // If rangeDiff is 1, then it is a continuous hash range. If not, there is a hole. + if (!rangeDiff.equals(BigInteger.ONE)) { + LOG.error("Incomplete hash range found between " + leftmostLeaseToReportInCaseOfHole + + " and " + rightLease); + return Optional.of(new HashRangeHole(leftmostLeaseToReportInCaseOfHole.getHashKeyRange(), + rightLease.getHashKeyRange())); + } + + leftmostLeaseToReportInCaseOfHole = rightLease; + leftLeaseHashRange = rightLeaseHashRange; + } + } + } + + return Optional.empty(); + } + + @VisibleForTesting + static List sortLeasesByHashRange(List leasesWithHashKeyRanges) { + if (leasesWithHashKeyRanges.size() == 0 || leasesWithHashKeyRanges.size() == 1) { + return leasesWithHashKeyRanges; + } + Collections.sort(leasesWithHashKeyRanges, new HashKeyRangeComparator()); + return leasesWithHashKeyRanges; + } + @Value + private static class HashRangeHole { + private final HashKeyRangeForLease hashRangeAtStartOfPossibleHole; + private final HashKeyRangeForLease hashRangeAtEndOfPossibleHole; + + HashRangeHole() { + hashRangeAtStartOfPossibleHole = hashRangeAtEndOfPossibleHole = null; + } + + HashRangeHole(HashKeyRangeForLease hashRangeAtStartOfPossibleHole, + HashKeyRangeForLease hashRangeAtEndOfPossibleHole) { + this.hashRangeAtStartOfPossibleHole = hashRangeAtStartOfPossibleHole; + this.hashRangeAtEndOfPossibleHole = hashRangeAtEndOfPossibleHole; + } + } + + private class HashRangeHoleTracker { + private HashRangeHole hashRangeHole; + @Getter + private Integer numConsecutiveHoles; + + public boolean hashHighConfidenceOfHoleWith(@NonNull HashRangeHole hashRangeHole) { + if (hashRangeHole.equals(this.hashRangeHole)) { + ++this.numConsecutiveHoles; + } else { + this.hashRangeHole = hashRangeHole; + this.numConsecutiveHoles = 1; + } + + return numConsecutiveHoles >= leasesRecoveryAuditorInconsistencyConfidenceThreshold; + } + + public void reset() { + this.hashRangeHole = null; + this.numConsecutiveHoles = 0; + } + } + + private static class HashKeyRangeComparator implements Comparator, Serializable { + private static final long serialVersionUID = 1L; + + @Override + public int compare(KinesisClientLease lease, KinesisClientLease otherLease) { + Validate.notNull(lease); + Validate.notNull(otherLease); + Validate.notNull(lease.getHashKeyRange()); + Validate.notNull(otherLease.getHashKeyRange()); + return ComparisonChain.start() + .compare(lease.getHashKeyRange().startingHashKey(), otherLease.getHashKeyRange().startingHashKey()) + .compare(lease.getHashKeyRange().endingHashKey(), otherLease.getHashKeyRange().endingHashKey()) + .result(); + } + } +} diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/MetricsCollectingTaskDecorator.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/MetricsCollectingTaskDecorator.java index 4f770313..b7d1b016 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/MetricsCollectingTaskDecorator.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/MetricsCollectingTaskDecorator.java @@ -21,7 +21,7 @@ import com.amazonaws.services.kinesis.metrics.interfaces.MetricsLevel; /** * Decorates an ITask and reports metrics about its timing and success/failure. */ -class MetricsCollectingTaskDecorator implements ITask { +public class MetricsCollectingTaskDecorator implements ITask { private final ITask other; private final IMetricsFactory factory; diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PeriodicShardSyncStrategy.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PeriodicShardSyncStrategy.java index c85fbbef..dce4bb02 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PeriodicShardSyncStrategy.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PeriodicShardSyncStrategy.java @@ -6,9 +6,9 @@ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; */ class PeriodicShardSyncStrategy implements ShardSyncStrategy { - private PeriodicShardSyncManager periodicShardSyncManager; + private IPeriodicShardSyncManager periodicShardSyncManager; - PeriodicShardSyncStrategy(PeriodicShardSyncManager periodicShardSyncManager) { + PeriodicShardSyncStrategy(IPeriodicShardSyncManager periodicShardSyncManager) { this.periodicShardSyncManager = periodicShardSyncManager; } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardEndShardSyncStrategy.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardEndShardSyncStrategy.java index 9efe2f51..d9b0810b 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardEndShardSyncStrategy.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardEndShardSyncStrategy.java @@ -17,10 +17,10 @@ class ShardEndShardSyncStrategy implements ShardSyncStrategy { private ShardSyncTaskManager shardSyncTaskManager; /** Runs periodic shard sync jobs in the background as an auditor process for shard-end syncs. */ - private PeriodicShardSyncManager periodicShardSyncManager; + private IPeriodicShardSyncManager periodicShardSyncManager; ShardEndShardSyncStrategy(ShardSyncTaskManager shardSyncTaskManager, - PeriodicShardSyncManager periodicShardSyncManager) { + IPeriodicShardSyncManager periodicShardSyncManager) { this.shardSyncTaskManager = shardSyncTaskManager; this.periodicShardSyncManager = periodicShardSyncManager; } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTask.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTask.java index 13c43b0e..5fafff66 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTask.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTask.java @@ -30,7 +30,7 @@ import java.util.List; * It will clean up leases/activities for shards that have been completely processed (if * cleanupLeasesUponShardCompletion is true). */ -class ShardSyncTask implements ITask { +public class ShardSyncTask implements ITask { private static final Log LOG = LogFactory.getLog(ShardSyncTask.class); @@ -56,7 +56,7 @@ class ShardSyncTask implements ITask { * @param shardSyncer shardSyncer instance used to check and create new leases * @param latestShards latest snapshot of shards to reuse */ - ShardSyncTask(IKinesisProxy kinesisProxy, + public ShardSyncTask(IKinesisProxy kinesisProxy, ILeaseManager leaseManager, InitialPositionInStreamExtended initialPositionInStream, boolean cleanupLeasesUponShardCompletion, diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/TaskResult.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/TaskResult.java index db22c97b..502d10f0 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/TaskResult.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/TaskResult.java @@ -22,7 +22,7 @@ import java.util.List; * Used to capture information from a task that we want to communicate back to the higher layer. * E.g. exception thrown when executing the task, if we reach end of a shard. */ -class TaskResult { +public class TaskResult { // Did we reach the end of the shard while processing this task. private boolean shardEndReached; @@ -38,7 +38,7 @@ class TaskResult { /** * @return the shardEndReached */ - protected boolean isShardEndReached() { + public boolean isShardEndReached() { return shardEndReached; } @@ -77,7 +77,7 @@ class TaskResult { /** * @param e Any exception encountered when running the process task. */ - TaskResult(Exception e) { + public TaskResult(Exception e) { this(e, false); } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java index a69ea6ca..a55d41e3 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java @@ -156,7 +156,7 @@ public class Worker implements Runnable { // Periodic Shard Sync related fields private LeaderDecider leaderDecider; private ShardSyncStrategy shardSyncStrategy; - private PeriodicShardSyncManager leaderElectedPeriodicShardSyncManager; + private IPeriodicShardSyncManager leaderElectedPeriodicShardSyncManager; private final LeaseCleanupManager leaseCleanupManager; @@ -533,7 +533,7 @@ public class Worker implements Runnable { IMetricsFactory metricsFactory, long taskBackoffTimeMillis, long failoverTimeMillis, boolean skipShardSyncAtWorkerInitializationIfLeasesExist, ShardPrioritization shardPrioritization, Optional retryGetRecordsInSeconds, Optional maxGetRecordsThreadPool, WorkerStateChangeListener workerStateChangeListener, - LeaseCleanupValidator leaseCleanupValidator, LeaderDecider leaderDecider, PeriodicShardSyncManager periodicShardSyncManager) { + LeaseCleanupValidator leaseCleanupValidator, LeaderDecider leaderDecider, IPeriodicShardSyncManager periodicShardSyncManager) { this(applicationName, recordProcessorFactory, config, streamConfig, initialPositionInStream, parentShardPollIntervalMillis, shardSyncIdleTimeMillis, cleanupLeasesUponShardCompletion, checkpoint, leaseCoordinator, execService, metricsFactory, taskBackoffTimeMillis, failoverTimeMillis, @@ -550,7 +550,7 @@ public class Worker implements Runnable { boolean skipShardSyncAtWorkerInitializationIfLeasesExist, ShardPrioritization shardPrioritization, Optional retryGetRecordsInSeconds, Optional maxGetRecordsThreadPool, WorkerStateChangeListener workerStateChangeListener, ShardSyncer shardSyncer, LeaderDecider leaderDecider, - PeriodicShardSyncManager periodicShardSyncManager) { + IPeriodicShardSyncManager periodicShardSyncManager) { this.applicationName = applicationName; this.recordProcessorFactory = recordProcessorFactory; this.config = config; @@ -590,7 +590,7 @@ public class Worker implements Runnable { */ private void createShardSyncStrategy(ShardSyncStrategyType strategyType, LeaderDecider leaderDecider, - PeriodicShardSyncManager periodicShardSyncManager) { + IPeriodicShardSyncManager periodicShardSyncManager) { switch (strategyType) { case PERIODIC: this.leaderDecider = getOrCreateLeaderDecider(leaderDecider); @@ -652,7 +652,7 @@ public class Worker implements Runnable { /** * @return the leaderElectedPeriodicShardSyncManager */ - PeriodicShardSyncManager getPeriodicShardSyncManager() { + IPeriodicShardSyncManager getPeriodicShardSyncManager() { return leaderElectedPeriodicShardSyncManager; } @@ -1224,7 +1224,7 @@ public class Worker implements Runnable { * KinesisClientLibConfiguration * @return Returns metrics factory based on the config. */ - private static IMetricsFactory getMetricsFactory(AmazonCloudWatch cloudWatchClient, + public static IMetricsFactory getMetricsFactory(AmazonCloudWatch cloudWatchClient, KinesisClientLibConfiguration config) { IMetricsFactory metricsFactory; if (config.getMetricsLevel() == MetricsLevel.NONE) { @@ -1278,13 +1278,13 @@ public class Worker implements Runnable { /** A non-null PeriodicShardSyncManager can only provided from unit tests. Any application code will create the * PeriodicShardSyncManager for the first time here. */ - private PeriodicShardSyncManager getOrCreatePeriodicShardSyncManager(PeriodicShardSyncManager periodicShardSyncManager, + private IPeriodicShardSyncManager getOrCreatePeriodicShardSyncManager(IPeriodicShardSyncManager periodicShardSyncManager, boolean isAuditorMode) { if (periodicShardSyncManager != null) { return periodicShardSyncManager; } - return new PeriodicShardSyncManager(config.getWorkerIdentifier(), + return new KinesisPeriodicShardSyncManager(config.getWorkerIdentifier(), leaderDecider, new ShardSyncTask(streamConfig.getStreamProxy(), leaseCoordinator.getLeaseManager(), @@ -1353,6 +1353,8 @@ public class Worker implements Runnable { @Setter @Accessors(fluent = true) private IKinesisProxy kinesisProxy; @Setter @Accessors(fluent = true) + private IPeriodicShardSyncManager periodicShardSyncManager; + @Setter @Accessors(fluent = true) private WorkerStateChangeListener workerStateChangeListener; @Setter @Accessors(fluent = true) private LeaseCleanupValidator leaseCleanupValidator; @@ -1421,6 +1423,12 @@ public class Worker implements Runnable { throw new IllegalArgumentException( "Kinesis Client Library configuration needs to be provided to build Worker"); } + if (periodicShardSyncManager != null) { + if (leaseManager == null || shardSyncer == null || metricsFactory == null || leaderDecider == null) { + + throw new IllegalArgumentException("LeaseManager, ShardSyncer, MetricsFactory, and LeaderDecider must be provided if PeriodicShardSyncManager is provided"); + } + } if (recordProcessorFactory == null) { throw new IllegalArgumentException("A Record Processor Factory needs to be provided to build Worker"); } @@ -1546,7 +1554,7 @@ public class Worker implements Runnable { workerStateChangeListener, shardSyncer, leaderDecider, - null /* PeriodicShardSyncManager */); + periodicShardSyncManager); } > R createClient(final T builder, From 372f98b21a91487e36612d528c56765a44b0aa86 Mon Sep 17 00:00:00 2001 From: gguptp <112846463+gguptp@users.noreply.github.com> Date: Thu, 6 Oct 2022 22:38:50 +0530 Subject: [PATCH 32/48] Every other change for DynamoDBStreamsKinesis Adapter Compatibility (#995) Co-authored-by: Nicholas Gutierrez --- ...ynchronousGetRecordsRetrievalStrategy.java | 10 +- .../lib/worker/BlockOnParentShardTask.java | 4 +- .../worker/GetRecordsRetrievalStrategy.java | 6 +- .../lib/worker/IDataFetcher.java | 23 +++ .../lib/worker/IShardConsumer.java | 25 +++ .../lib/worker/IShardConsumerFactory.java | 34 ++++ .../lib/worker/InitializeTask.java | 8 +- .../worker/KinesisClientLibConfiguration.java | 6 +- .../KinesisClientLibLeaseCoordinator.java | 4 +- ...States.java => KinesisConsumerStates.java} | 60 +++---- .../lib/worker/KinesisDataFetcher.java | 9 +- ...onsumer.java => KinesisShardConsumer.java} | 70 ++++---- .../worker/KinesisShardConsumerFactory.java | 48 +++++ ...downTask.java => KinesisShutdownTask.java} | 29 ++-- .../lib/worker/PeriodicShardSyncManager.java | 2 +- .../lib/worker/PrefetchGetRecordsCache.java | 2 +- .../clientlibrary/lib/worker/ProcessTask.java | 8 +- .../worker/RecordProcessorCheckpointer.java | 12 +- .../lib/worker/SequenceNumberValidator.java | 2 +- .../lib/worker/ShutdownNotificationTask.java | 4 +- .../lib/worker/ShutdownReason.java | 6 +- .../lib/worker/StreamConfig.java | 8 +- ...ynchronousGetRecordsRetrievalStrategy.java | 4 +- .../clientlibrary/lib/worker/Worker.java | 45 +++-- .../leases/impl/LeaseCleanupManager.java | 2 +- .../lib/worker/ConsumerStatesTest.java | 16 +- .../GracefulShutdownCoordinatorTest.java | 2 +- .../lib/worker/ShardConsumerTest.java | 164 +++++++++--------- .../lib/worker/ShutdownTaskTest.java | 28 +-- .../clientlibrary/lib/worker/WorkerTest.java | 40 ++--- 30 files changed, 408 insertions(+), 273 deletions(-) create mode 100644 src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/IDataFetcher.java create mode 100644 src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/IShardConsumer.java create mode 100644 src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/IShardConsumerFactory.java rename src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/{ConsumerStates.java => KinesisConsumerStates.java} (90%) rename src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/{ShardConsumer.java => KinesisShardConsumer.java} (92%) create mode 100644 src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisShardConsumerFactory.java rename src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/{ShutdownTask.java => KinesisShutdownTask.java} (99%) diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/AsynchronousGetRecordsRetrievalStrategy.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/AsynchronousGetRecordsRetrievalStrategy.java index 7fb16073..47990226 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/AsynchronousGetRecordsRetrievalStrategy.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/AsynchronousGetRecordsRetrievalStrategy.java @@ -45,24 +45,24 @@ public class AsynchronousGetRecordsRetrievalStrategy implements GetRecordsRetrie private static final int TIME_TO_KEEP_ALIVE = 5; private static final int CORE_THREAD_POOL_COUNT = 1; - private final KinesisDataFetcher dataFetcher; + private final IDataFetcher dataFetcher; private final ExecutorService executorService; private final int retryGetRecordsInSeconds; private final String shardId; final Supplier> completionServiceSupplier; - public AsynchronousGetRecordsRetrievalStrategy(@NonNull final KinesisDataFetcher dataFetcher, + public AsynchronousGetRecordsRetrievalStrategy(@NonNull final IDataFetcher dataFetcher, final int retryGetRecordsInSeconds, final int maxGetRecordsThreadPool, String shardId) { this(dataFetcher, buildExector(maxGetRecordsThreadPool, shardId), retryGetRecordsInSeconds, shardId); } - public AsynchronousGetRecordsRetrievalStrategy(final KinesisDataFetcher dataFetcher, + public AsynchronousGetRecordsRetrievalStrategy(final IDataFetcher dataFetcher, final ExecutorService executorService, final int retryGetRecordsInSeconds, String shardId) { this(dataFetcher, executorService, retryGetRecordsInSeconds, () -> new ExecutorCompletionService<>(executorService), shardId); } - AsynchronousGetRecordsRetrievalStrategy(KinesisDataFetcher dataFetcher, ExecutorService executorService, + AsynchronousGetRecordsRetrievalStrategy(IDataFetcher dataFetcher, ExecutorService executorService, int retryGetRecordsInSeconds, Supplier> completionServiceSupplier, String shardId) { this.dataFetcher = dataFetcher; @@ -148,7 +148,7 @@ public class AsynchronousGetRecordsRetrievalStrategy implements GetRecordsRetrie } @Override - public KinesisDataFetcher getDataFetcher() { + public IDataFetcher getDataFetcher() { return dataFetcher; } } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/BlockOnParentShardTask.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/BlockOnParentShardTask.java index 4f4c49b8..2f001b2e 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/BlockOnParentShardTask.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/BlockOnParentShardTask.java @@ -30,7 +30,7 @@ import com.amazonaws.services.kinesis.leases.interfaces.ILeaseManager; * If we don't find a checkpoint for the parent shard(s), we assume they have been trimmed and directly * proceed with processing data from the shard. */ -class BlockOnParentShardTask implements ITask { +public class BlockOnParentShardTask implements ITask { private static final Log LOG = LogFactory.getLog(BlockOnParentShardTask.class); private final ShardInfo shardInfo; @@ -45,7 +45,7 @@ class BlockOnParentShardTask implements ITask { * @param leaseManager Used to fetch the lease and checkpoint info for parent shards * @param parentShardPollIntervalMillis Sleep time if the parent shard has not completed processing */ - BlockOnParentShardTask(ShardInfo shardInfo, + public BlockOnParentShardTask(ShardInfo shardInfo, ILeaseManager leaseManager, long parentShardPollIntervalMillis) { this.shardInfo = shardInfo; diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/GetRecordsRetrievalStrategy.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/GetRecordsRetrievalStrategy.java index 71a15340..d831a4f1 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/GetRecordsRetrievalStrategy.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/GetRecordsRetrievalStrategy.java @@ -46,9 +46,9 @@ public interface GetRecordsRetrievalStrategy { boolean isShutdown(); /** - * Returns the KinesisDataFetcher used to getRecords from Kinesis. + * Returns the IDataFetcher used to getRecords * - * @return KinesisDataFetcher + * @return IDataFetcher */ - KinesisDataFetcher getDataFetcher(); + IDataFetcher getDataFetcher(); } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/IDataFetcher.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/IDataFetcher.java new file mode 100644 index 00000000..29f088d1 --- /dev/null +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/IDataFetcher.java @@ -0,0 +1,23 @@ +package com.amazonaws.services.kinesis.clientlibrary.lib.worker; + +import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber; +import com.amazonaws.services.kinesis.model.ChildShard; + +import java.util.List; + +public interface IDataFetcher { + + DataFetcherResult getRecords(int maxRecords); + + void initialize(String initialCheckpoint, InitialPositionInStreamExtended initialPositionInStream); + + void initialize(ExtendedSequenceNumber initialCheckpoint, InitialPositionInStreamExtended initialPositionInStream); + + void advanceIteratorTo(String sequenceNumber, InitialPositionInStreamExtended initialPositionInStream); + + void restartIterator(); + + boolean isShardEndReached(); + + List getChildShards(); +} diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/IShardConsumer.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/IShardConsumer.java new file mode 100644 index 00000000..ac2e31d1 --- /dev/null +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/IShardConsumer.java @@ -0,0 +1,25 @@ +package com.amazonaws.services.kinesis.clientlibrary.lib.worker; + +public interface IShardConsumer { + + boolean isSkipShardSyncAtWorkerInitializationIfLeasesExist(); + + enum TaskOutcome { + SUCCESSFUL, END_OF_SHARD, NOT_COMPLETE, FAILURE, LEASE_NOT_FOUND + } + + boolean consumeShard(); + + boolean isShutdown(); + + ShutdownReason getShutdownReason(); + + boolean beginShutdown(); + + void notifyShutdownRequested(ShutdownNotification shutdownNotification); + + KinesisConsumerStates.ShardConsumerState getCurrentState(); + + boolean isShutdownRequested(); + +} diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/IShardConsumerFactory.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/IShardConsumerFactory.java new file mode 100644 index 00000000..acb23018 --- /dev/null +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/IShardConsumerFactory.java @@ -0,0 +1,34 @@ +package com.amazonaws.services.kinesis.clientlibrary.lib.worker; + +import com.amazonaws.services.kinesis.clientlibrary.interfaces.ICheckpoint; +import com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessor; +import com.amazonaws.services.kinesis.leases.impl.LeaseCleanupManager; +import com.amazonaws.services.kinesis.metrics.interfaces.IMetricsFactory; + +import java.util.Optional; +import java.util.concurrent.ExecutorService; + +public interface IShardConsumerFactory { + + /** + * Returns a shard consumer to be used for consuming a (assigned) shard. + * + * @return Returns a shard consumer object. + */ + IShardConsumer createShardConsumer(ShardInfo shardInfo, + StreamConfig streamConfig, + ICheckpoint checkpointTracker, + IRecordProcessor recordProcessor, + RecordProcessorCheckpointer recordProcessorCheckpointer, + KinesisClientLibLeaseCoordinator leaseCoordinator, + long parentShardPollIntervalMillis, + boolean cleanupLeasesUponShardCompletion, + ExecutorService executorService, + IMetricsFactory metricsFactory, + long taskBackoffTimeMillis, + boolean skipShardSyncAtWorkerInitializationIfLeasesExist, + Optional retryGetRecordsInSeconds, + Optional maxGetRecordsThreadPool, + KinesisClientLibConfiguration config, ShardSyncer shardSyncer, ShardSyncStrategy shardSyncStrategy, + LeaseCleanupManager leaseCleanupManager); +} \ No newline at end of file diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/InitializeTask.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/InitializeTask.java index 5343470f..0b4fa651 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/InitializeTask.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/InitializeTask.java @@ -29,7 +29,7 @@ import com.amazonaws.services.kinesis.metrics.interfaces.MetricsLevel; /** * Task for initializing shard position and invoking the RecordProcessor initialize() API. */ -class InitializeTask implements ITask { +public class InitializeTask implements ITask { private static final Log LOG = LogFactory.getLog(InitializeTask.class); @@ -37,7 +37,7 @@ class InitializeTask implements ITask { private final ShardInfo shardInfo; private final IRecordProcessor recordProcessor; - private final KinesisDataFetcher dataFetcher; + private final IDataFetcher dataFetcher; private final TaskType taskType = TaskType.INITIALIZE; private final ICheckpoint checkpoint; private final RecordProcessorCheckpointer recordProcessorCheckpointer; @@ -49,11 +49,11 @@ class InitializeTask implements ITask { /** * Constructor. */ - InitializeTask(ShardInfo shardInfo, + public InitializeTask(ShardInfo shardInfo, IRecordProcessor recordProcessor, ICheckpoint checkpoint, RecordProcessorCheckpointer recordProcessorCheckpointer, - KinesisDataFetcher dataFetcher, + IDataFetcher dataFetcher, long backoffTimeMillis, StreamConfig streamConfig, GetRecordsCache getRecordsCache) { diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java index ccde83f3..7aa2f885 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java @@ -61,7 +61,7 @@ public class KinesisClientLibConfiguration { public static final int DEFAULT_MAX_RECORDS = 10000; /** - * The default value for how long the {@link ShardConsumer} should sleep if no records are returned from the call to + * The default value for how long the {@link KinesisShardConsumer} should sleep if no records are returned from the call to * {@link com.amazonaws.services.kinesis.AmazonKinesis#getRecords(com.amazonaws.services.kinesis.model.GetRecordsRequest)}. */ public static final long DEFAULT_IDLETIME_BETWEEN_READS_MILLIS = 1000L; @@ -91,7 +91,7 @@ public class KinesisClientLibConfiguration { public static final boolean DEFAULT_CLEANUP_LEASES_UPON_SHARDS_COMPLETION = true; /** - * Interval to run lease cleanup thread in {@link LeaseCleanupManager}. + * Interval to run lease cleanup thread in {@link com.amazonaws.services.kinesis.leases.impl.LeaseCleanupManager}. */ private static final long DEFAULT_LEASE_CLEANUP_INTERVAL_MILLIS = Duration.ofMinutes(1).toMillis(); @@ -1030,7 +1030,7 @@ public class KinesisClientLibConfiguration { * Keeping it protected to forbid outside callers from depending on this internal object. * @return The initialPositionInStreamExtended object. */ - protected InitialPositionInStreamExtended getInitialPositionInStreamExtended() { + public InitialPositionInStreamExtended getInitialPositionInStreamExtended() { return initialPositionInStreamExtended; } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibLeaseCoordinator.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibLeaseCoordinator.java index 47baad04..ad55ab52 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibLeaseCoordinator.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibLeaseCoordinator.java @@ -51,7 +51,7 @@ import com.amazonaws.services.kinesis.metrics.interfaces.IMetricsFactory; /** * This class is used to coordinate/manage leases owned by this worker process and to get/set checkpoints. */ -class KinesisClientLibLeaseCoordinator extends LeaseCoordinator implements ICheckpoint { +public class KinesisClientLibLeaseCoordinator extends LeaseCoordinator implements ICheckpoint { private static final Log LOG = LogFactory.getLog(KinesisClientLibLeaseCoordinator.class); @@ -368,7 +368,7 @@ class KinesisClientLibLeaseCoordinator extends LeaseCoordinator getLeaseManager() { + public ILeaseManager getLeaseManager() { return leaseManager; } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ConsumerStates.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisConsumerStates.java similarity index 90% rename from src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ConsumerStates.java rename to src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisConsumerStates.java index 5cf55dbf..766d6fba 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ConsumerStates.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisConsumerStates.java @@ -15,7 +15,7 @@ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; /** - * Top level container for all the possible states a {@link ShardConsumer} can be in. The logic for creation of tasks, + * Top level container for all the possible states a {@link KinesisShardConsumer} can be in. The logic for creation of tasks, * and state transitions is contained within the {@link ConsumerState} objects. * *

State Diagram

@@ -64,12 +64,12 @@ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; * +-------------------+ * */ -class ConsumerStates { +public class KinesisConsumerStates { /** * Enumerates processing states when working on a shard. */ - enum ShardConsumerState { + public enum ShardConsumerState { // @formatter:off WAITING_ON_PARENT_SHARDS(new BlockedOnParentState()), INITIALIZING(new InitializingState()), @@ -96,7 +96,7 @@ class ConsumerStates { * do when a transition occurs. * */ - interface ConsumerState { + public interface ConsumerState { /** * Creates a new task for this state using the passed in consumer to build the task. If there is no task * required for this state it may return a null value. {@link ConsumerState}'s are allowed to modify the @@ -106,11 +106,11 @@ class ConsumerStates { * the consumer to use build the task, or execute state. * @return a valid task for this state or null if there is no task required. */ - ITask createTask(ShardConsumer consumer); + ITask createTask(KinesisShardConsumer consumer); /** * Provides the next state of the consumer upon success of the task return by - * {@link ConsumerState#createTask(ShardConsumer)}. + * {@link ConsumerState#createTask(KinesisShardConsumer)}. * * @return the next state that the consumer should transition to, this may be the same object as the current * state. @@ -129,7 +129,7 @@ class ConsumerStates { ConsumerState shutdownTransition(ShutdownReason shutdownReason); /** - * The type of task that {@link ConsumerState#createTask(ShardConsumer)} would return. This is always a valid state + * The type of task that {@link ConsumerState#createTask(KinesisShardConsumer)} would return. This is always a valid state * even if createTask would return a null value. * * @return the type of task that this state represents. @@ -149,7 +149,7 @@ class ConsumerStates { } /** - * The initial state that any {@link ShardConsumer} should start in. + * The initial state that any {@link KinesisShardConsumer} should start in. */ static final ConsumerState INITIAL_STATE = ShardConsumerState.WAITING_ON_PARENT_SHARDS.getConsumerState(); @@ -187,7 +187,7 @@ class ConsumerStates { static class BlockedOnParentState implements ConsumerState { @Override - public ITask createTask(ShardConsumer consumer) { + public ITask createTask(KinesisShardConsumer consumer) { return new BlockOnParentShardTask(consumer.getShardInfo(), consumer.getLeaseManager(), consumer.getParentShardPollIntervalMillis()); } @@ -247,10 +247,10 @@ class ConsumerStates { * * */ - static class InitializingState implements ConsumerState { + public static class InitializingState implements ConsumerState { @Override - public ITask createTask(ShardConsumer consumer) { + public ITask createTask(KinesisShardConsumer consumer) { return new InitializeTask(consumer.getShardInfo(), consumer.getRecordProcessor(), consumer.getCheckpoint(), @@ -311,7 +311,7 @@ class ConsumerStates { static class ProcessingState implements ConsumerState { @Override - public ITask createTask(ShardConsumer consumer) { + public ITask createTask(KinesisShardConsumer consumer) { return new ProcessTask(consumer.getShardInfo(), consumer.getStreamConfig(), consumer.getRecordProcessor(), @@ -358,10 +358,10 @@ class ConsumerStates { *

Valid Transitions

*
*
Success
- *
Success shouldn't normally be called since the {@link ShardConsumer} is marked for shutdown.
+ *
Success shouldn't normally be called since the {@link KinesisShardConsumer} is marked for shutdown.
*
Shutdown
*
At this point records are being retrieved, and processed. An explicit shutdown will allow the record - * processor one last chance to checkpoint, and then the {@link ShardConsumer} will be held in an idle state. + * processor one last chance to checkpoint, and then the {@link KinesisShardConsumer} will be held in an idle state. *
*
{@link ShutdownReason#REQUESTED}
*
Remains in the {@link ShardConsumerState#SHUTDOWN_REQUESTED}, but the state implementation changes to @@ -377,7 +377,7 @@ class ConsumerStates { static class ShutdownNotificationState implements ConsumerState { @Override - public ITask createTask(ShardConsumer consumer) { + public ITask createTask(KinesisShardConsumer consumer) { return new ShutdownNotificationTask(consumer.getRecordProcessor(), consumer.getRecordProcessorCheckpointer(), consumer.getShutdownNotification(), @@ -414,24 +414,24 @@ class ConsumerStates { } /** - * Once the {@link ShutdownNotificationState} has been completed the {@link ShardConsumer} must not re-enter any of the - * processing states. This state idles the {@link ShardConsumer} until the worker triggers the final shutdown state. + * Once the {@link ShutdownNotificationState} has been completed the {@link KinesisShardConsumer} must not re-enter any of the + * processing states. This state idles the {@link KinesisShardConsumer} until the worker triggers the final shutdown state. * *

Valid Transitions

*
*
Success
*
*

- * Success shouldn't normally be called since the {@link ShardConsumer} is marked for shutdown. + * Success shouldn't normally be called since the {@link KinesisShardConsumer} is marked for shutdown. *

*

* Remains in the {@link ShutdownNotificationCompletionState} *

*
*
Shutdown
- *
At this point the {@link ShardConsumer} has notified the record processor of the impending shutdown, and is + *
At this point the {@link KinesisShardConsumer} has notified the record processor of the impending shutdown, and is * waiting that notification. While waiting for the notification no further processing should occur on the - * {@link ShardConsumer}. + * {@link KinesisShardConsumer}. *
*
{@link ShutdownReason#REQUESTED}
*
Remains in the {@link ShardConsumerState#SHUTDOWN_REQUESTED}, and the state implementation remains @@ -447,7 +447,7 @@ class ConsumerStates { static class ShutdownNotificationCompletionState implements ConsumerState { @Override - public ITask createTask(ShardConsumer consumer) { + public ITask createTask(KinesisShardConsumer consumer) { return null; } @@ -481,14 +481,14 @@ class ConsumerStates { } /** - * This state is entered if the {@link ShardConsumer} loses its lease, or reaches the end of the shard. + * This state is entered if the {@link KinesisShardConsumer} loses its lease, or reaches the end of the shard. * *

Valid Transitions

*
*
Success
*
*

- * Success shouldn't normally be called since the {@link ShardConsumer} is marked for shutdown. + * Success shouldn't normally be called since the {@link KinesisShardConsumer} is marked for shutdown. *

*

* Transitions to the {@link ShutdownCompleteState} @@ -497,7 +497,7 @@ class ConsumerStates { *

Shutdown
*
At this point the record processor has processed the final shutdown indication, and depending on the shutdown * reason taken the correct course of action. From this point on there should be no more interactions with the - * record processor or {@link ShardConsumer}. + * record processor or {@link KinesisShardConsumer}. *
*
{@link ShutdownReason#REQUESTED}
*
@@ -519,8 +519,8 @@ class ConsumerStates { static class ShuttingDownState implements ConsumerState { @Override - public ITask createTask(ShardConsumer consumer) { - return new ShutdownTask(consumer.getShardInfo(), + public ITask createTask(KinesisShardConsumer consumer) { + return new KinesisShutdownTask(consumer.getShardInfo(), consumer.getRecordProcessor(), consumer.getRecordProcessorCheckpointer(), consumer.getShutdownReason(), @@ -562,21 +562,21 @@ class ConsumerStates { } /** - * This is the final state for the {@link ShardConsumer}. This occurs once all shutdown activities are completed. + * This is the final state for the {@link KinesisShardConsumer}. This occurs once all shutdown activities are completed. * *

Valid Transitions

*
*
Success
*
*

- * Success shouldn't normally be called since the {@link ShardConsumer} is marked for shutdown. + * Success shouldn't normally be called since the {@link KinesisShardConsumer} is marked for shutdown. *

*

* Remains in the {@link ShutdownCompleteState} *

*
*
Shutdown
- *
At this point the all shutdown activites are completed, and the {@link ShardConsumer} should not take any + *
At this point the all shutdown activites are completed, and the {@link KinesisShardConsumer} should not take any * further actions. *
*
{@link ShutdownReason#REQUESTED}
@@ -599,7 +599,7 @@ class ConsumerStates { static class ShutdownCompleteState implements ConsumerState { @Override - public ITask createTask(ShardConsumer consumer) { + public ITask createTask(KinesisShardConsumer consumer) { if (consumer.getShutdownNotification() != null) { consumer.getShutdownNotification().shutdownComplete(); } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisDataFetcher.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisDataFetcher.java index ae4e321d..4c8d638b 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisDataFetcher.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisDataFetcher.java @@ -39,7 +39,7 @@ import lombok.Data; /** * Used to get data from Amazon Kinesis. Tracks iterator state internally. */ -class KinesisDataFetcher { +public class KinesisDataFetcher implements IDataFetcher{ private static final Log LOG = LogFactory.getLog(KinesisDataFetcher.class); @@ -185,7 +185,7 @@ class KinesisDataFetcher { * @param sequenceNumber advance the iterator to the record at this sequence number. * @param initialPositionInStream The initialPositionInStream. */ - void advanceIteratorTo(String sequenceNumber, InitialPositionInStreamExtended initialPositionInStream) { + public void advanceIteratorTo(String sequenceNumber, InitialPositionInStreamExtended initialPositionInStream) { if (sequenceNumber == null) { throw new IllegalArgumentException("SequenceNumber should not be null: shardId " + shardId); } else if (sequenceNumber.equals(SentinelCheckpoint.LATEST.toString())) { @@ -276,11 +276,11 @@ class KinesisDataFetcher { /** * @return the shardEndReached */ - protected boolean isShardEndReached() { + public boolean isShardEndReached() { return isShardEndReached; } - protected List getChildShards() { + public List getChildShards() { return childShards; } @@ -290,5 +290,4 @@ class KinesisDataFetcher { String getNextIterator() { return nextIterator; } - } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumer.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisShardConsumer.java similarity index 92% rename from src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumer.java rename to src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisShardConsumer.java index 11ee39b5..b8e227c4 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumer.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisShardConsumer.java @@ -14,7 +14,6 @@ */ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; - import java.util.List; import java.util.Optional; import java.util.concurrent.ExecutorService; @@ -35,7 +34,6 @@ import com.amazonaws.services.kinesis.leases.impl.KinesisClientLease; import com.amazonaws.services.kinesis.leases.interfaces.ILeaseManager; import com.amazonaws.services.kinesis.metrics.interfaces.IMetricsFactory; import com.google.common.annotations.VisibleForTesting; - import lombok.Getter; /** @@ -43,9 +41,9 @@ import lombok.Getter; * The instance should be shutdown when we lose the primary responsibility for a shard. * A new instance should be created if the primary responsibility is reassigned back to this process. */ -class ShardConsumer { +public class KinesisShardConsumer implements IShardConsumer{ - private static final Log LOG = LogFactory.getLog(ShardConsumer.class); + private static final Log LOG = LogFactory.getLog(KinesisShardConsumer.class); private final StreamConfig streamConfig; private final IRecordProcessor recordProcessor; @@ -78,7 +76,7 @@ class ShardConsumer { @Getter private final GetRecordsCache getRecordsCache; - private static final GetRecordsRetrievalStrategy makeStrategy(KinesisDataFetcher dataFetcher, + private static final GetRecordsRetrievalStrategy makeStrategy(IDataFetcher dataFetcher, Optional retryGetRecordsInSeconds, Optional maxGetRecordsThreadPool, ShardInfo shardInfo) { @@ -93,7 +91,7 @@ class ShardConsumer { * Tracks current state. It is only updated via the consumeStream/shutdown APIs. Therefore we don't do * much coordination/synchronization to handle concurrent reads/updates. */ - private ConsumerStates.ConsumerState currentState = ConsumerStates.INITIAL_STATE; + private KinesisConsumerStates.ConsumerState currentState = KinesisConsumerStates.INITIAL_STATE; /* * Used to track if we lost the primary responsibility. Once set to true, we will start shutting down. * If we regain primary responsibility before shutdown is complete, Worker should create a new ShardConsumer object. @@ -116,7 +114,7 @@ class ShardConsumer { */ // CHECKSTYLE:IGNORE ParameterNumber FOR NEXT 10 LINES @Deprecated - ShardConsumer(ShardInfo shardInfo, + KinesisShardConsumer(ShardInfo shardInfo, StreamConfig streamConfig, ICheckpoint checkpoint, IRecordProcessor recordProcessor, @@ -162,7 +160,7 @@ class ShardConsumer { */ // CHECKSTYLE:IGNORE ParameterNumber FOR NEXT 10 LINES @Deprecated - ShardConsumer(ShardInfo shardInfo, + KinesisShardConsumer(ShardInfo shardInfo, StreamConfig streamConfig, ICheckpoint checkpoint, IRecordProcessor recordProcessor, @@ -223,7 +221,7 @@ class ShardConsumer { * @param shardSyncer shardSyncer instance used to check and create new leases */ @Deprecated - ShardConsumer(ShardInfo shardInfo, + KinesisShardConsumer(ShardInfo shardInfo, StreamConfig streamConfig, ICheckpoint checkpoint, IRecordProcessor recordProcessor, @@ -269,23 +267,23 @@ class ShardConsumer { * @param shardSyncer shardSyncer instance used to check and create new leases * @param leaseCleanupManager used to clean up leases in lease table. */ - ShardConsumer(ShardInfo shardInfo, - StreamConfig streamConfig, - ICheckpoint checkpoint, - IRecordProcessor recordProcessor, - RecordProcessorCheckpointer recordProcessorCheckpointer, - KinesisClientLibLeaseCoordinator leaseCoordinator, - long parentShardPollIntervalMillis, - boolean cleanupLeasesOfCompletedShards, - ExecutorService executorService, - IMetricsFactory metricsFactory, - long backoffTimeMillis, - boolean skipShardSyncAtWorkerInitializationIfLeasesExist, - KinesisDataFetcher kinesisDataFetcher, - Optional retryGetRecordsInSeconds, - Optional maxGetRecordsThreadPool, - KinesisClientLibConfiguration config, ShardSyncer shardSyncer, ShardSyncStrategy shardSyncStrategy, - LeaseCleanupManager leaseCleanupManager) { + KinesisShardConsumer(ShardInfo shardInfo, + StreamConfig streamConfig, + ICheckpoint checkpoint, + IRecordProcessor recordProcessor, + RecordProcessorCheckpointer recordProcessorCheckpointer, + KinesisClientLibLeaseCoordinator leaseCoordinator, + long parentShardPollIntervalMillis, + boolean cleanupLeasesOfCompletedShards, + ExecutorService executorService, + IMetricsFactory metricsFactory, + long backoffTimeMillis, + boolean skipShardSyncAtWorkerInitializationIfLeasesExist, + KinesisDataFetcher kinesisDataFetcher, + Optional retryGetRecordsInSeconds, + Optional maxGetRecordsThreadPool, + KinesisClientLibConfiguration config, ShardSyncer shardSyncer, ShardSyncStrategy shardSyncStrategy, + LeaseCleanupManager leaseCleanupManager) { this.shardInfo = shardInfo; this.streamConfig = streamConfig; this.checkpoint = checkpoint; @@ -314,7 +312,7 @@ class ShardConsumer { * * @return true if a new process task was submitted, false otherwise */ - synchronized boolean consumeShard() { + public synchronized boolean consumeShard() { return checkAndSubmitNextTask(); } @@ -373,10 +371,6 @@ class ShardConsumer { return skipShardSyncAtWorkerInitializationIfLeasesExist; } - private enum TaskOutcome { - SUCCESSFUL, END_OF_SHARD, NOT_COMPLETE, FAILURE, LEASE_NOT_FOUND - } - private TaskOutcome determineTaskOutcome() { try { TaskResult result = future.get(); @@ -423,7 +417,7 @@ class ShardConsumer { * * @param shutdownNotification used to signal that the record processor has been given the chance to shutdown. */ - void notifyShutdownRequested(ShutdownNotification shutdownNotification) { + public void notifyShutdownRequested(ShutdownNotification shutdownNotification) { this.shutdownNotification = shutdownNotification; markForShutdown(ShutdownReason.REQUESTED); } @@ -434,7 +428,7 @@ class ShardConsumer { * * @return true if shutdown is complete (false if shutdown is still in progress) */ - synchronized boolean beginShutdown() { + public synchronized boolean beginShutdown() { markForShutdown(ShutdownReason.ZOMBIE); checkAndSubmitNextTask(); @@ -454,14 +448,14 @@ class ShardConsumer { * * @return true if shutdown is complete */ - boolean isShutdown() { + public boolean isShutdown() { return currentState.isTerminal(); } /** * @return the shutdownReason */ - ShutdownReason getShutdownReason() { + public ShutdownReason getShutdownReason() { return shutdownReason; } @@ -497,7 +491,7 @@ class ShardConsumer { } if (isShutdownRequested() && taskOutcome != TaskOutcome.FAILURE) { currentState = currentState.shutdownTransition(shutdownReason); - } else if (isShutdownRequested() && ConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS.equals(currentState.getState())) { + } else if (isShutdownRequested() && KinesisConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS.equals(currentState.getState())) { currentState = currentState.shutdownTransition(shutdownReason); } else if (taskOutcome == TaskOutcome.SUCCESSFUL) { if (currentState.getTaskType() == currentTask.getTaskType()) { @@ -516,7 +510,7 @@ class ShardConsumer { } @VisibleForTesting - boolean isShutdownRequested() { + public boolean isShutdownRequested() { return shutdownReason != null; } @@ -525,7 +519,7 @@ class ShardConsumer { * * @return the currentState */ - ConsumerStates.ShardConsumerState getCurrentState() { + public KinesisConsumerStates.ShardConsumerState getCurrentState() { return currentState.getState(); } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisShardConsumerFactory.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisShardConsumerFactory.java new file mode 100644 index 00000000..bbcae852 --- /dev/null +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisShardConsumerFactory.java @@ -0,0 +1,48 @@ +package com.amazonaws.services.kinesis.clientlibrary.lib.worker; + +import com.amazonaws.services.kinesis.clientlibrary.interfaces.ICheckpoint; +import com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessor; +import com.amazonaws.services.kinesis.leases.impl.LeaseCleanupManager; +import com.amazonaws.services.kinesis.metrics.interfaces.IMetricsFactory; + +import java.util.Optional; +import java.util.concurrent.ExecutorService; + +public class KinesisShardConsumerFactory implements IShardConsumerFactory{ + + @Override + public IShardConsumer createShardConsumer(ShardInfo shardInfo, + StreamConfig streamConfig, + ICheckpoint checkpointTracker, + IRecordProcessor recordProcessor, + RecordProcessorCheckpointer recordProcessorCheckpointer, + KinesisClientLibLeaseCoordinator leaseCoordinator, + long parentShardPollIntervalMillis, + boolean cleanupLeasesUponShardCompletion, + ExecutorService executorService, + IMetricsFactory metricsFactory, + long taskBackoffTimeMillis, + boolean skipShardSyncAtWorkerInitializationIfLeasesExist, + Optional retryGetRecordsInSeconds, + Optional maxGetRecordsThreadPool, + KinesisClientLibConfiguration config, ShardSyncer shardSyncer, ShardSyncStrategy shardSyncStrategy, + LeaseCleanupManager leaseCleanupManager) { + return new KinesisShardConsumer(shardInfo, + streamConfig, + checkpointTracker, + recordProcessor, + recordProcessorCheckpointer, + leaseCoordinator, + parentShardPollIntervalMillis, + cleanupLeasesUponShardCompletion, + executorService, + metricsFactory, + taskBackoffTimeMillis, + skipShardSyncAtWorkerInitializationIfLeasesExist, + new KinesisDataFetcher(streamConfig.getStreamProxy(), shardInfo), + retryGetRecordsInSeconds, + maxGetRecordsThreadPool, + config, shardSyncer, shardSyncStrategy, + leaseCleanupManager); + } +} diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTask.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisShutdownTask.java similarity index 99% rename from src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTask.java rename to src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisShutdownTask.java index 07c9fff2..a444f26f 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTask.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisShutdownTask.java @@ -14,25 +14,24 @@ */ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; -import com.amazonaws.services.kinesis.leases.LeasePendingDeletion; import com.amazonaws.services.kinesis.clientlibrary.exceptions.internal.BlockedOnParentShardException; -import com.amazonaws.services.kinesis.leases.exceptions.CustomerApplicationException; -import com.amazonaws.services.kinesis.leases.exceptions.DependencyException; -import com.amazonaws.services.kinesis.leases.exceptions.InvalidStateException; -import com.amazonaws.services.kinesis.leases.exceptions.ProvisionedThroughputException; -import com.amazonaws.services.kinesis.leases.impl.LeaseCleanupManager; -import com.amazonaws.services.kinesis.leases.impl.UpdateField; -import com.amazonaws.services.kinesis.model.ChildShard; -import com.amazonaws.util.CollectionUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - import com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessor; import com.amazonaws.services.kinesis.clientlibrary.proxies.IKinesisProxy; import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber; import com.amazonaws.services.kinesis.clientlibrary.types.ShutdownInput; +import com.amazonaws.services.kinesis.leases.LeasePendingDeletion; +import com.amazonaws.services.kinesis.leases.exceptions.CustomerApplicationException; +import com.amazonaws.services.kinesis.leases.exceptions.DependencyException; +import com.amazonaws.services.kinesis.leases.exceptions.InvalidStateException; +import com.amazonaws.services.kinesis.leases.exceptions.ProvisionedThroughputException; import com.amazonaws.services.kinesis.leases.impl.KinesisClientLease; +import com.amazonaws.services.kinesis.leases.impl.LeaseCleanupManager; +import com.amazonaws.services.kinesis.leases.impl.UpdateField; +import com.amazonaws.services.kinesis.model.ChildShard; +import com.amazonaws.util.CollectionUtils; import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import java.util.List; import java.util.Objects; @@ -44,9 +43,9 @@ import java.util.stream.Collectors; /** * Task for invoking the RecordProcessor shutdown() callback. */ -class ShutdownTask implements ITask { +public class KinesisShutdownTask implements ITask { - private static final Log LOG = LogFactory.getLog(ShutdownTask.class); + private static final Log LOG = LogFactory.getLog(KinesisShutdownTask.class); @VisibleForTesting static final int RETRY_RANDOM_MAX_RANGE = 50; @@ -72,7 +71,7 @@ class ShutdownTask implements ITask { * Constructor. */ // CHECKSTYLE:IGNORE ParameterNumber FOR NEXT 10 LINES - ShutdownTask(ShardInfo shardInfo, + KinesisShutdownTask(ShardInfo shardInfo, IRecordProcessor recordProcessor, RecordProcessorCheckpointer recordProcessorCheckpointer, ShutdownReason reason, diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PeriodicShardSyncManager.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PeriodicShardSyncManager.java index cdf73e82..35b07b9d 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PeriodicShardSyncManager.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PeriodicShardSyncManager.java @@ -407,4 +407,4 @@ class PeriodicShardSyncManager { .result(); } } -} +} \ No newline at end of file diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PrefetchGetRecordsCache.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PrefetchGetRecordsCache.java index a4cf74d8..3e4dbcb7 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PrefetchGetRecordsCache.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PrefetchGetRecordsCache.java @@ -60,7 +60,7 @@ public class PrefetchGetRecordsCache implements GetRecordsCache { private PrefetchCounters prefetchCounters; private boolean started = false; private final String operation; - private final KinesisDataFetcher dataFetcher; + private final IDataFetcher dataFetcher; private final String shardId; /** diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ProcessTask.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ProcessTask.java index cd543e23..61587450 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ProcessTask.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ProcessTask.java @@ -41,7 +41,7 @@ import com.amazonaws.services.kinesis.model.Shard; /** * Task for fetching data records and invoking processRecords() on the record processor instance. */ -class ProcessTask implements ITask { +public class ProcessTask implements ITask { private static final Log LOG = LogFactory.getLog(ProcessTask.class); @@ -55,7 +55,7 @@ class ProcessTask implements ITask { private final ShardInfo shardInfo; private final IRecordProcessor recordProcessor; private final RecordProcessorCheckpointer recordProcessorCheckpointer; - private final KinesisDataFetcher dataFetcher; + private final IDataFetcher dataFetcher; private final TaskType taskType = TaskType.PROCESS; private final StreamConfig streamConfig; private final long backoffTimeMillis; @@ -81,7 +81,7 @@ class ProcessTask implements ITask { * The retrieval strategy for fetching records from kinesis */ public ProcessTask(ShardInfo shardInfo, StreamConfig streamConfig, IRecordProcessor recordProcessor, - RecordProcessorCheckpointer recordProcessorCheckpointer, KinesisDataFetcher dataFetcher, + RecordProcessorCheckpointer recordProcessorCheckpointer, IDataFetcher dataFetcher, long backoffTimeMillis, boolean skipShardSyncAtWorkerInitializationIfLeasesExist, GetRecordsCache getRecordsCache) { this(shardInfo, streamConfig, recordProcessor, recordProcessorCheckpointer, dataFetcher, backoffTimeMillis, @@ -107,7 +107,7 @@ class ProcessTask implements ITask { * determines how throttling events should be reported in the log. */ public ProcessTask(ShardInfo shardInfo, StreamConfig streamConfig, IRecordProcessor recordProcessor, - RecordProcessorCheckpointer recordProcessorCheckpointer, KinesisDataFetcher dataFetcher, + RecordProcessorCheckpointer recordProcessorCheckpointer, IDataFetcher dataFetcher, long backoffTimeMillis, boolean skipShardSyncAtWorkerInitializationIfLeasesExist, ThrottlingReporter throttlingReporter, GetRecordsCache getRecordsCache) { super(); diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/RecordProcessorCheckpointer.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/RecordProcessorCheckpointer.java index dbee9218..49c1b0e5 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/RecordProcessorCheckpointer.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/RecordProcessorCheckpointer.java @@ -37,7 +37,7 @@ import com.amazonaws.services.kinesis.model.Record; * The Amazon Kinesis Client Library will instantiate an object and provide a reference to the application * RecordProcessor instance. Amazon Kinesis Client Library will create one instance per shard assignment. */ -class RecordProcessorCheckpointer implements IRecordProcessorCheckpointer { +public class RecordProcessorCheckpointer implements IRecordProcessorCheckpointer { private static final Log LOG = LogFactory.getLog(RecordProcessorCheckpointer.class); @@ -62,7 +62,7 @@ class RecordProcessorCheckpointer implements IRecordProcessorCheckpointer { * @param checkpoint Used to checkpoint progress of a RecordProcessor * @param validator Used for validating sequence numbers */ - RecordProcessorCheckpointer(ShardInfo shardInfo, + public RecordProcessorCheckpointer(ShardInfo shardInfo, ICheckpoint checkpoint, SequenceNumberValidator validator, IMetricsFactory metricsFactory) { @@ -231,7 +231,7 @@ class RecordProcessorCheckpointer implements IRecordProcessorCheckpointer { /** * @return the lastCheckpointValue */ - ExtendedSequenceNumber getLastCheckpointValue() { + public ExtendedSequenceNumber getLastCheckpointValue() { return lastCheckpointValue; } @@ -244,14 +244,14 @@ class RecordProcessorCheckpointer implements IRecordProcessorCheckpointer { * * @return the largest permitted checkpoint */ - synchronized ExtendedSequenceNumber getLargestPermittedCheckpointValue() { + public synchronized ExtendedSequenceNumber getLargestPermittedCheckpointValue() { return largestPermittedCheckpointValue; } /** * @param largestPermittedCheckpointValue the largest permitted checkpoint */ - synchronized void setLargestPermittedCheckpointValue(ExtendedSequenceNumber largestPermittedCheckpointValue) { + public synchronized void setLargestPermittedCheckpointValue(ExtendedSequenceNumber largestPermittedCheckpointValue) { this.largestPermittedCheckpointValue = largestPermittedCheckpointValue; } @@ -262,7 +262,7 @@ class RecordProcessorCheckpointer implements IRecordProcessorCheckpointer { * * @param extendedSequenceNumber */ - synchronized void setSequenceNumberAtShardEnd(ExtendedSequenceNumber extendedSequenceNumber) { + public synchronized void setSequenceNumberAtShardEnd(ExtendedSequenceNumber extendedSequenceNumber) { this.sequenceNumberAtShardEnd = extendedSequenceNumber; } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/SequenceNumberValidator.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/SequenceNumberValidator.java index 3ca28235..ac81dc8b 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/SequenceNumberValidator.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/SequenceNumberValidator.java @@ -51,7 +51,7 @@ public class SequenceNumberValidator { * @param validateWithGetIterator Whether to attempt to get an iterator for this shard id and the sequence numbers * being validated */ - SequenceNumberValidator(IKinesisProxy proxy, String shardId, boolean validateWithGetIterator) { + public SequenceNumberValidator(IKinesisProxy proxy, String shardId, boolean validateWithGetIterator) { this.proxy = proxy; this.shardId = shardId; this.validateWithGetIterator = validateWithGetIterator; diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownNotificationTask.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownNotificationTask.java index ecf5041f..98e4702e 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownNotificationTask.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownNotificationTask.java @@ -21,14 +21,14 @@ import com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IShutdownNotif /** * Notifies record processor of incoming shutdown request, and gives them a chance to checkpoint. */ -class ShutdownNotificationTask implements ITask { +public class ShutdownNotificationTask implements ITask { private final IRecordProcessor recordProcessor; private final IRecordProcessorCheckpointer recordProcessorCheckpointer; private final ShutdownNotification shutdownNotification; private final ShardInfo shardInfo; - ShutdownNotificationTask(IRecordProcessor recordProcessor, IRecordProcessorCheckpointer recordProcessorCheckpointer, ShutdownNotification shutdownNotification, ShardInfo shardInfo) { + public ShutdownNotificationTask(IRecordProcessor recordProcessor, IRecordProcessorCheckpointer recordProcessorCheckpointer, ShutdownNotification shutdownNotification, ShardInfo shardInfo) { this.recordProcessor = recordProcessor; this.recordProcessorCheckpointer = recordProcessorCheckpointer; this.shutdownNotification = shutdownNotification; diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownReason.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownReason.java index 5e29d6dd..c326e361 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownReason.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownReason.java @@ -15,8 +15,8 @@ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; import com.amazonaws.services.kinesis.clientlibrary.types.ShutdownInput; -import static com.amazonaws.services.kinesis.clientlibrary.lib.worker.ConsumerStates.ConsumerState; -import static com.amazonaws.services.kinesis.clientlibrary.lib.worker.ConsumerStates.ShardConsumerState; +import static com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisConsumerStates.ConsumerState; +import static com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisConsumerStates.ShardConsumerState; /** @@ -72,7 +72,7 @@ public enum ShutdownReason { return reason.rank > this.rank; } - ConsumerState getShutdownState() { + public ConsumerState getShutdownState() { return shutdownState; } } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/StreamConfig.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/StreamConfig.java index fff6b71f..7ef5be9c 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/StreamConfig.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/StreamConfig.java @@ -19,7 +19,7 @@ import com.amazonaws.services.kinesis.clientlibrary.proxies.IKinesisProxy; /** * Used to capture stream configuration and pass it along. */ -class StreamConfig { +public class StreamConfig { private final IKinesisProxy streamProxy; private final int maxRecords; @@ -54,7 +54,7 @@ class StreamConfig { /** * @return the streamProxy */ - IKinesisProxy getStreamProxy() { + public IKinesisProxy getStreamProxy() { return streamProxy; } @@ -82,14 +82,14 @@ class StreamConfig { /** * @return the initialPositionInStream */ - InitialPositionInStreamExtended getInitialPositionInStream() { + public InitialPositionInStreamExtended getInitialPositionInStream() { return initialPositionInStream; } /** * @return validateSequenceNumberBeforeCheckpointing */ - boolean shouldValidateSequenceNumberBeforeCheckpointing() { + public boolean shouldValidateSequenceNumberBeforeCheckpointing() { return validateSequenceNumberBeforeCheckpointing; } } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/SynchronousGetRecordsRetrievalStrategy.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/SynchronousGetRecordsRetrievalStrategy.java index 5da33ac6..ac46be33 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/SynchronousGetRecordsRetrievalStrategy.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/SynchronousGetRecordsRetrievalStrategy.java @@ -24,7 +24,7 @@ import lombok.NonNull; @Data public class SynchronousGetRecordsRetrievalStrategy implements GetRecordsRetrievalStrategy { @NonNull - private final KinesisDataFetcher dataFetcher; + private final IDataFetcher dataFetcher; @Override public GetRecordsResult getRecords(final int maxRecords) { @@ -44,7 +44,7 @@ public class SynchronousGetRecordsRetrievalStrategy implements GetRecordsRetriev } @Override - public KinesisDataFetcher getDataFetcher() { + public IDataFetcher getDataFetcher() { return dataFetcher; } } diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java index a55d41e3..e751c4b3 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java @@ -137,7 +137,7 @@ public class Worker implements Runnable { // Holds consumers for shards the worker is currently tracking. Key is shard // info, value is ShardConsumer. - private ConcurrentMap shardInfoShardConsumerMap = new ConcurrentHashMap(); + private ConcurrentMap shardInfoShardConsumerMap = new ConcurrentHashMap(); private final boolean cleanupLeasesUponShardCompletion; private final boolean skipShardSyncAtWorkerInitializationIfLeasesExist; @@ -160,6 +160,9 @@ public class Worker implements Runnable { private final LeaseCleanupManager leaseCleanupManager; + // Shard Consumer Factory + private IShardConsumerFactory shardConsumerFactory; + /** * Constructor. * @@ -539,7 +542,7 @@ public class Worker implements Runnable { leaseCoordinator, execService, metricsFactory, taskBackoffTimeMillis, failoverTimeMillis, skipShardSyncAtWorkerInitializationIfLeasesExist, shardPrioritization, retryGetRecordsInSeconds, maxGetRecordsThreadPool, workerStateChangeListener, new KinesisShardSyncer(leaseCleanupValidator), - leaderDecider, periodicShardSyncManager); + leaderDecider, periodicShardSyncManager, null /*ShardConsumerFactory*/); } Worker(String applicationName, IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration config, @@ -550,7 +553,7 @@ public class Worker implements Runnable { boolean skipShardSyncAtWorkerInitializationIfLeasesExist, ShardPrioritization shardPrioritization, Optional retryGetRecordsInSeconds, Optional maxGetRecordsThreadPool, WorkerStateChangeListener workerStateChangeListener, ShardSyncer shardSyncer, LeaderDecider leaderDecider, - IPeriodicShardSyncManager periodicShardSyncManager) { + IPeriodicShardSyncManager periodicShardSyncManager, IShardConsumerFactory shardConsumerFactory) { this.applicationName = applicationName; this.recordProcessorFactory = recordProcessorFactory; this.config = config; @@ -580,6 +583,7 @@ public class Worker implements Runnable { Executors.newSingleThreadScheduledExecutor(), metricsFactory, cleanupLeasesUponShardCompletion, config.leaseCleanupIntervalMillis(), config.completedLeaseCleanupThresholdMillis(), config.garbageLeaseCleanupThresholdMillis(), config.getMaxRecords()); + this.shardConsumerFactory = shardConsumerFactory; } /** @@ -687,7 +691,7 @@ public class Worker implements Runnable { boolean foundCompletedShard = false; Set assignedShards = new HashSet<>(); for (ShardInfo shardInfo : getShardInfoForAssignments()) { - ShardConsumer shardConsumer = createOrGetShardConsumer(shardInfo, recordProcessorFactory); + IShardConsumer shardConsumer = createOrGetShardConsumer(shardInfo, recordProcessorFactory); if (shardConsumer.isShutdown() && shardConsumer.getShutdownReason().equals(ShutdownReason.TERMINATE)) { foundCompletedShard = true; } else { @@ -983,9 +987,9 @@ public class Worker implements Runnable { ShutdownNotification shutdownNotification = new ShardConsumerShutdownNotification(leaseCoordinator, lease, notificationCompleteLatch, shutdownCompleteLatch); ShardInfo shardInfo = KinesisClientLibLeaseCoordinator.convertLeaseToAssignment(lease); - ShardConsumer consumer = shardInfoShardConsumerMap.get(shardInfo); + IShardConsumer consumer = shardInfoShardConsumerMap.get(shardInfo); - if (consumer == null || ConsumerStates.ShardConsumerState.SHUTDOWN_COMPLETE.equals(consumer.getCurrentState())) { + if (consumer == null || KinesisConsumerStates.ShardConsumerState.SHUTDOWN_COMPLETE.equals(consumer.getCurrentState())) { // // CASE1: There is a race condition between retrieving the current assignments, and creating the // notification. If the a lease is lost in between these two points, we explicitly decrement the @@ -1007,7 +1011,7 @@ public class Worker implements Runnable { return shutdownComplete; } - ConcurrentMap getShardInfoShardConsumerMap() { + ConcurrentMap getShardInfoShardConsumerMap() { return shardInfoShardConsumerMap; } @@ -1107,8 +1111,8 @@ public class Worker implements Runnable { * RecordProcessor factory * @return ShardConsumer for the shard */ - ShardConsumer createOrGetShardConsumer(ShardInfo shardInfo, IRecordProcessorFactory processorFactory) { - ShardConsumer consumer = shardInfoShardConsumerMap.get(shardInfo); + IShardConsumer createOrGetShardConsumer(ShardInfo shardInfo, IRecordProcessorFactory processorFactory) { + IShardConsumer consumer = shardInfoShardConsumerMap.get(shardInfo); // Instantiate a new consumer if we don't have one, or the one we // had was from an earlier // lease instance (and was shutdown). Don't need to create another @@ -1123,7 +1127,7 @@ public class Worker implements Runnable { return consumer; } - protected ShardConsumer buildConsumer(ShardInfo shardInfo, IRecordProcessorFactory processorFactory) { + protected IShardConsumer buildConsumer(ShardInfo shardInfo, IRecordProcessorFactory processorFactory) { final IRecordProcessor recordProcessor = processorFactory.createProcessor(); final RecordProcessorCheckpointer recordProcessorCheckpointer = new RecordProcessorCheckpointer( shardInfo, @@ -1134,7 +1138,11 @@ public class Worker implements Runnable { streamConfig.shouldValidateSequenceNumberBeforeCheckpointing()), metricsFactory); - return new ShardConsumer(shardInfo, + if(shardConsumerFactory == null){ //Default to KinesisShardConsumerFactory if null + this.shardConsumerFactory = new KinesisShardConsumerFactory(); + } + + return shardConsumerFactory.createShardConsumer(shardInfo, streamConfig, checkpointTracker, recordProcessor, @@ -1146,7 +1154,6 @@ public class Worker implements Runnable { metricsFactory, taskBackoffTimeMillis, skipShardSyncAtWorkerInitializationIfLeasesExist, - new KinesisDataFetcher(streamConfig.getStreamProxy(), shardInfo), retryGetRecordsInSeconds, maxGetRecordsThreadPool, config, shardSyncer, shardSyncStrategy, @@ -1225,7 +1232,7 @@ public class Worker implements Runnable { * @return Returns metrics factory based on the config. */ public static IMetricsFactory getMetricsFactory(AmazonCloudWatch cloudWatchClient, - KinesisClientLibConfiguration config) { + KinesisClientLibConfiguration config) { IMetricsFactory metricsFactory; if (config.getMetricsLevel() == MetricsLevel.NONE) { metricsFactory = new NullMetricsFactory(); @@ -1355,6 +1362,8 @@ public class Worker implements Runnable { @Setter @Accessors(fluent = true) private IPeriodicShardSyncManager periodicShardSyncManager; @Setter @Accessors(fluent = true) + private IShardConsumerFactory shardConsumerFactory; + @Setter @Accessors(fluent = true) private WorkerStateChangeListener workerStateChangeListener; @Setter @Accessors(fluent = true) private LeaseCleanupValidator leaseCleanupValidator; @@ -1429,6 +1438,10 @@ public class Worker implements Runnable { throw new IllegalArgumentException("LeaseManager, ShardSyncer, MetricsFactory, and LeaderDecider must be provided if PeriodicShardSyncManager is provided"); } } + if(shardConsumerFactory == null){ + shardConsumerFactory = new KinesisShardConsumerFactory(); + } + if (recordProcessorFactory == null) { throw new IllegalArgumentException("A Record Processor Factory needs to be provided to build Worker"); } @@ -1511,7 +1524,7 @@ public class Worker implements Runnable { } // We expect users to either inject both LeaseRenewer and the corresponding thread-pool, or neither of them (DEFAULT). - if (leaseRenewer == null) { + if (leaseRenewer == null) { ExecutorService leaseRenewerThreadPool = LeaseCoordinator.getDefaultLeaseRenewalExecutorService(config.getMaxLeaseRenewalThreads()); leaseRenewer = new LeaseRenewer<>(leaseManager, config.getWorkerIdentifier(), config.getFailoverTimeMillis(), leaseRenewerThreadPool); } @@ -1520,7 +1533,6 @@ public class Worker implements Runnable { leaderDecider = new DeterministicShuffleShardSyncLeaderDecider(leaseManager, Executors.newSingleThreadScheduledExecutor(), PERIODIC_SHARD_SYNC_MAX_WORKERS_DEFAULT); } - return new Worker(config.getApplicationName(), recordProcessorFactory, config, @@ -1554,7 +1566,8 @@ public class Worker implements Runnable { workerStateChangeListener, shardSyncer, leaderDecider, - periodicShardSyncManager); + periodicShardSyncManager /*PeriodicShardSyncManager*/, + shardConsumerFactory); } > R createClient(final T builder, diff --git a/src/main/java/com/amazonaws/services/kinesis/leases/impl/LeaseCleanupManager.java b/src/main/java/com/amazonaws/services/kinesis/leases/impl/LeaseCleanupManager.java index d19fc3ed..ab254961 100644 --- a/src/main/java/com/amazonaws/services/kinesis/leases/impl/LeaseCleanupManager.java +++ b/src/main/java/com/amazonaws/services/kinesis/leases/impl/LeaseCleanupManager.java @@ -406,4 +406,4 @@ public class LeaseCleanupManager { boolean cleanedUp; String failureMsg; } -} +} \ No newline at end of file diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ConsumerStatesTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ConsumerStatesTest.java index 6a5e76b9..63cceb6e 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ConsumerStatesTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ConsumerStatesTest.java @@ -14,8 +14,8 @@ */ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; -import static com.amazonaws.services.kinesis.clientlibrary.lib.worker.ConsumerStates.ConsumerState; -import static com.amazonaws.services.kinesis.clientlibrary.lib.worker.ConsumerStates.ShardConsumerState; +import static com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisConsumerStates.ConsumerState; +import static com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisConsumerStates.ShardConsumerState; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.MatcherAssert.assertThat; @@ -50,7 +50,7 @@ import com.amazonaws.services.kinesis.leases.interfaces.ILeaseManager; public class ConsumerStatesTest { @Mock - private ShardConsumer consumer; + private KinesisShardConsumer consumer; @Mock private StreamConfig streamConfig; @Mock @@ -251,9 +251,9 @@ public class ConsumerStatesTest { equalTo((IRecordProcessorCheckpointer) recordProcessorCheckpointer))); assertThat(task, shutdownReqTask(ShutdownNotification.class, "shutdownNotification", equalTo(shutdownNotification))); - assertThat(state.successTransition(), equalTo(ConsumerStates.SHUTDOWN_REQUEST_COMPLETION_STATE)); + assertThat(state.successTransition(), equalTo(KinesisConsumerStates.SHUTDOWN_REQUEST_COMPLETION_STATE)); assertThat(state.shutdownTransition(ShutdownReason.REQUESTED), - equalTo(ConsumerStates.SHUTDOWN_REQUEST_COMPLETION_STATE)); + equalTo(KinesisConsumerStates.SHUTDOWN_REQUEST_COMPLETION_STATE)); assertThat(state.shutdownTransition(ShutdownReason.ZOMBIE), equalTo(ShardConsumerState.SHUTTING_DOWN.getConsumerState())); assertThat(state.shutdownTransition(ShutdownReason.TERMINATE), @@ -266,7 +266,7 @@ public class ConsumerStatesTest { @Test public void shutdownRequestCompleteStateTest() { - ConsumerState state = ConsumerStates.SHUTDOWN_REQUEST_COMPLETION_STATE; + ConsumerState state = KinesisConsumerStates.SHUTDOWN_REQUEST_COMPLETION_STATE; assertThat(state.createTask(consumer), nullValue()); @@ -345,9 +345,9 @@ public class ConsumerStatesTest { verify(shutdownNotification, never()).shutdownComplete(); } - static ReflectionPropertyMatcher shutdownTask(Class valueTypeClass, + static ReflectionPropertyMatcher shutdownTask(Class valueTypeClass, String propertyName, Matcher matcher) { - return taskWith(ShutdownTask.class, valueTypeClass, propertyName, matcher); + return taskWith(KinesisShutdownTask.class, valueTypeClass, propertyName, matcher); } static ReflectionPropertyMatcher shutdownReqTask( diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/GracefulShutdownCoordinatorTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/GracefulShutdownCoordinatorTest.java index 11e274cb..2a3b2774 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/GracefulShutdownCoordinatorTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/GracefulShutdownCoordinatorTest.java @@ -49,7 +49,7 @@ public class GracefulShutdownCoordinatorTest { @Mock private Callable contextCallable; @Mock - private ConcurrentMap shardInfoConsumerMap; + private ConcurrentMap shardInfoConsumerMap; @Test public void testAllShutdownCompletedAlready() throws Exception { diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumerTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumerTest.java index 7a5e7fd2..8936a28e 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumerTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumerTest.java @@ -85,7 +85,7 @@ import com.amazonaws.services.kinesis.model.Shard; import com.amazonaws.services.kinesis.model.ShardIteratorType; /** - * Unit tests of {@link ShardConsumer}. + * Unit tests of {@link KinesisShardConsumer}. */ @RunWith(MockitoJUnitRunner.class) public class ShardConsumerTest { @@ -160,8 +160,8 @@ public class ShardConsumerTest { callProcessRecordsForEmptyRecordList, skipCheckpointValidationValue, INITIAL_POSITION_LATEST); - ShardConsumer consumer = - new ShardConsumer(shardInfo, + KinesisShardConsumer consumer = + new KinesisShardConsumer(shardInfo, streamConfig, checkpoint, processor, @@ -175,19 +175,19 @@ public class ShardConsumerTest { config, shardSyncer, shardSyncStrategy); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); consumer.consumeShard(); // initialize Thread.sleep(50L); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); consumer.consumeShard(); // initialize Thread.sleep(50L); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.INITIALIZING))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.INITIALIZING))); consumer.consumeShard(); // initialize Thread.sleep(50L); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.INITIALIZING))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.INITIALIZING))); consumer.consumeShard(); // initialize Thread.sleep(50L); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.INITIALIZING))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.INITIALIZING))); } /** @@ -210,8 +210,8 @@ public class ShardConsumerTest { callProcessRecordsForEmptyRecordList, skipCheckpointValidationValue, INITIAL_POSITION_LATEST); - ShardConsumer consumer = - new ShardConsumer(shardInfo, + KinesisShardConsumer consumer = + new KinesisShardConsumer(shardInfo, streamConfig, checkpoint, processor, @@ -226,21 +226,21 @@ public class ShardConsumerTest { shardSyncer, shardSyncStrategy); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); consumer.consumeShard(); // initialize Thread.sleep(50L); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); doThrow(new RejectedExecutionException()).when(spyExecutorService).submit(any(InitializeTask.class)); consumer.consumeShard(); // initialize Thread.sleep(50L); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.INITIALIZING))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.INITIALIZING))); consumer.consumeShard(); // initialize Thread.sleep(50L); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.INITIALIZING))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.INITIALIZING))); consumer.consumeShard(); // initialize Thread.sleep(50L); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.INITIALIZING))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.INITIALIZING))); } @Test @@ -258,8 +258,8 @@ public class ShardConsumerTest { callProcessRecordsForEmptyRecordList, skipCheckpointValidationValue, INITIAL_POSITION_LATEST); - ShardConsumer consumer = - new ShardConsumer(shardInfo, + KinesisShardConsumer consumer = + new KinesisShardConsumer(shardInfo, streamConfig, checkpoint, processor, @@ -273,19 +273,19 @@ public class ShardConsumerTest { config, shardSyncer, shardSyncStrategy); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); consumer.consumeShard(); Thread.sleep(50L); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); consumer.consumeShard(); Thread.sleep(50L); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.INITIALIZING))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.INITIALIZING))); consumer.consumeShard(); Thread.sleep(50L); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.SHUTTING_DOWN))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.SHUTTING_DOWN))); consumer.consumeShard(); Thread.sleep(50L); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.SHUTDOWN_COMPLETE))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.SHUTDOWN_COMPLETE))); } @@ -300,8 +300,8 @@ public class ShardConsumerTest { callProcessRecordsForEmptyRecordList, skipCheckpointValidationValue, INITIAL_POSITION_LATEST); - ShardConsumer consumer = - new ShardConsumer(shardInfo, + KinesisShardConsumer consumer = + new KinesisShardConsumer(shardInfo, streamConfig, checkpoint, processor, @@ -324,10 +324,10 @@ public class ShardConsumerTest { when(checkpoint.getCheckpointObject(anyString())).thenReturn( new Checkpoint(checkpointSequenceNumber, pendingCheckpointSequenceNumber)); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); consumer.consumeShard(); // submit BlockOnParentShardTask Thread.sleep(50L); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); verify(processor, times(0)).initialize(any(InitializationInput.class)); // Throw Error when IRecordProcessor.initialize() is invoked. @@ -335,7 +335,7 @@ public class ShardConsumerTest { consumer.consumeShard(); // submit InitializeTask Thread.sleep(50L); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.INITIALIZING))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.INITIALIZING))); verify(processor, times(1)).initialize(argThat( initializationInputMatcher(checkpointSequenceNumber, pendingCheckpointSequenceNumber))); @@ -347,7 +347,7 @@ public class ShardConsumerTest { assertThat(e.getCause(), instanceOf(ExecutionException.class)); } Thread.sleep(50L); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.INITIALIZING))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.INITIALIZING))); verify(processor, times(1)).initialize(argThat( initializationInputMatcher(checkpointSequenceNumber, pendingCheckpointSequenceNumber))); @@ -355,7 +355,7 @@ public class ShardConsumerTest { consumer.consumeShard(); // submit InitializeTask again. Thread.sleep(50L); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.INITIALIZING))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.INITIALIZING))); verify(processor, times(2)).initialize(argThat( initializationInputMatcher(checkpointSequenceNumber, pendingCheckpointSequenceNumber))); verify(processor, times(2)).initialize(any(InitializationInput.class)); // no other calls with different args @@ -363,11 +363,11 @@ public class ShardConsumerTest { // Checking the status of submitted InitializeTask from above should pass. consumer.consumeShard(); Thread.sleep(50L); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.PROCESSING))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.PROCESSING))); } /** - * Test method for {@link ShardConsumer#consumeShard()} + * Test method for {@link KinesisShardConsumer#consumeShard()} */ @Test public final void testConsumeShard() throws Exception { @@ -420,8 +420,8 @@ public class ShardConsumerTest { any(IMetricsFactory.class), anyInt())) .thenReturn(getRecordsCache); - ShardConsumer consumer = - new ShardConsumer(shardInfo, + KinesisShardConsumer consumer = + new KinesisShardConsumer(shardInfo, streamConfig, checkpoint, processor, @@ -440,11 +440,11 @@ public class ShardConsumerTest { shardSyncer, shardSyncStrategy); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); consumer.consumeShard(); // check on parent shards Thread.sleep(50L); consumer.consumeShard(); // start initialization - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.INITIALIZING))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.INITIALIZING))); consumer.consumeShard(); // initialize processor.getInitializeLatch().await(5, TimeUnit.SECONDS); verify(getRecordsCache).start(); @@ -454,7 +454,7 @@ public class ShardConsumerTest { boolean newTaskSubmitted = consumer.consumeShard(); if (newTaskSubmitted) { LOG.debug("New processing task was submitted, call # " + i); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.PROCESSING))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.PROCESSING))); // CHECKSTYLE:IGNORE ModifiedControlVariable FOR NEXT 1 LINES i += maxRecords; } @@ -469,21 +469,21 @@ public class ShardConsumerTest { assertThat(processor.getNotifyShutdownLatch().await(1, TimeUnit.SECONDS), is(true)); Thread.sleep(50); assertThat(consumer.getShutdownReason(), equalTo(ShutdownReason.REQUESTED)); - assertThat(consumer.getCurrentState(), equalTo(ConsumerStates.ShardConsumerState.SHUTDOWN_REQUESTED)); + assertThat(consumer.getCurrentState(), equalTo(KinesisConsumerStates.ShardConsumerState.SHUTDOWN_REQUESTED)); verify(shutdownNotification).shutdownNotificationComplete(); assertThat(processor.isShutdownNotificationCalled(), equalTo(true)); consumer.consumeShard(); Thread.sleep(50); - assertThat(consumer.getCurrentState(), equalTo(ConsumerStates.ShardConsumerState.SHUTDOWN_REQUESTED)); + assertThat(consumer.getCurrentState(), equalTo(KinesisConsumerStates.ShardConsumerState.SHUTDOWN_REQUESTED)); consumer.beginShutdown(); Thread.sleep(50L); assertThat(consumer.getShutdownReason(), equalTo(ShutdownReason.ZOMBIE)); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.SHUTTING_DOWN))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.SHUTTING_DOWN))); consumer.beginShutdown(); consumer.consumeShard(); verify(shutdownNotification, atLeastOnce()).shutdownComplete(); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.SHUTDOWN_COMPLETE))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.SHUTDOWN_COMPLETE))); assertThat(processor.getShutdownReason(), is(equalTo(ShutdownReason.ZOMBIE))); verify(getRecordsCache).shutdown(); @@ -524,8 +524,8 @@ public class ShardConsumerTest { when(recordProcessorCheckpointer.getLastCheckpointValue()).thenReturn(ExtendedSequenceNumber.SHARD_END); when(streamConfig.getStreamProxy()).thenReturn(streamProxy); - final ShardConsumer consumer = - new ShardConsumer(shardInfo, + final KinesisShardConsumer consumer = + new KinesisShardConsumer(shardInfo, streamConfig, checkpoint, processor, @@ -544,21 +544,21 @@ public class ShardConsumerTest { shardSyncer, shardSyncStrategy); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); verify(parentLease, times(0)).getCheckpoint(); consumer.consumeShard(); // check on parent shards Thread.sleep(parentShardPollIntervalMillis * 2); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); verify(parentLease, times(1)).getCheckpoint(); consumer.notifyShutdownRequested(shutdownNotification); verify(shutdownNotification, times(0)).shutdownComplete(); assertThat(consumer.getShutdownReason(), equalTo(ShutdownReason.REQUESTED)); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); consumer.consumeShard(); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.SHUTTING_DOWN))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.SHUTTING_DOWN))); Thread.sleep(50L); consumer.beginShutdown(); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.SHUTDOWN_COMPLETE))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.SHUTDOWN_COMPLETE))); assertThat(consumer.isShutdown(), is(true)); verify(shutdownNotification, times(1)).shutdownComplete(); consumer.beginShutdown(); @@ -583,7 +583,7 @@ public class ShardConsumerTest { } /** - * Test method for {@link ShardConsumer#consumeShard()} that ensures a transient error thrown from the record + * Test method for {@link KinesisShardConsumer#consumeShard()} that ensures a transient error thrown from the record * processor's shutdown method with reason zombie will be retried. */ @Test @@ -646,8 +646,8 @@ public class ShardConsumerTest { metricsFactory ); - ShardConsumer consumer = - new ShardConsumer(shardInfo, + KinesisShardConsumer consumer = + new KinesisShardConsumer(shardInfo, streamConfig, checkpoint, processor, @@ -667,11 +667,11 @@ public class ShardConsumerTest { shardSyncStrategy); when(leaseCoordinator.updateLease(any(KinesisClientLease.class), any(UUID.class))).thenReturn(true); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); consumer.consumeShard(); // check on parent shards Thread.sleep(50L); consumer.consumeShard(); // start initialization - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.INITIALIZING))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.INITIALIZING))); consumer.consumeShard(); // initialize processor.getInitializeLatch().await(5, TimeUnit.SECONDS); verify(getRecordsCache).start(); @@ -681,7 +681,7 @@ public class ShardConsumerTest { boolean newTaskSubmitted = consumer.consumeShard(); if (newTaskSubmitted) { LOG.debug("New processing task was submitted, call # " + i); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.PROCESSING))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.PROCESSING))); // CHECKSTYLE:IGNORE ModifiedControlVariable FOR NEXT 1 LINES i += maxRecords; } @@ -709,12 +709,12 @@ public class ShardConsumerTest { // Wait for shutdown complete now that terminate shutdown is successful for (int i = 0; i < 100; i++) { consumer.consumeShard(); - if (consumer.getCurrentState() == ConsumerStates.ShardConsumerState.SHUTDOWN_COMPLETE) { + if (consumer.getCurrentState() == KinesisConsumerStates.ShardConsumerState.SHUTDOWN_COMPLETE) { break; } Thread.sleep(50L); } - assertThat(consumer.getCurrentState(), equalTo(ConsumerStates.ShardConsumerState.SHUTDOWN_COMPLETE)); + assertThat(consumer.getCurrentState(), equalTo(KinesisConsumerStates.ShardConsumerState.SHUTDOWN_COMPLETE)); assertThat(processor.getShutdownReason(), is(equalTo(ShutdownReason.TERMINATE))); @@ -732,7 +732,7 @@ public class ShardConsumerTest { /** - * Test method for {@link ShardConsumer#consumeShard()} that ensures the shardConsumer gets shutdown with shutdown + * Test method for {@link KinesisShardConsumer#consumeShard()} that ensures the shardConsumer gets shutdown with shutdown * reason TERMINATE when the shard end is reached. */ @Test @@ -795,8 +795,8 @@ public class ShardConsumerTest { metricsFactory ); - ShardConsumer consumer = - new ShardConsumer(shardInfo, + KinesisShardConsumer consumer = + new KinesisShardConsumer(shardInfo, streamConfig, checkpoint, processor, @@ -818,11 +818,11 @@ public class ShardConsumerTest { when(leaseCoordinator.getCurrentlyHeldLease(shardInfo.getShardId())).thenReturn(currentLease); when(leaseCoordinator.updateLease(any(KinesisClientLease.class), any(UUID.class))).thenReturn(true); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); consumer.consumeShard(); // check on parent shards Thread.sleep(50L); consumer.consumeShard(); // start initialization - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.INITIALIZING))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.INITIALIZING))); consumer.consumeShard(); // initialize processor.getInitializeLatch().await(5, TimeUnit.SECONDS); verify(getRecordsCache).start(); @@ -832,7 +832,7 @@ public class ShardConsumerTest { boolean newTaskSubmitted = consumer.consumeShard(); if (newTaskSubmitted) { LOG.debug("New processing task was submitted, call # " + i); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.PROCESSING))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.PROCESSING))); // CHECKSTYLE:IGNORE ModifiedControlVariable FOR NEXT 1 LINES i += maxRecords; } @@ -860,12 +860,12 @@ public class ShardConsumerTest { // Wait for shutdown complete now that terminate shutdown is successful for (int i = 0; i < 100; i++) { consumer.consumeShard(); - if (consumer.getCurrentState() == ConsumerStates.ShardConsumerState.SHUTDOWN_COMPLETE) { + if (consumer.getCurrentState() == KinesisConsumerStates.ShardConsumerState.SHUTDOWN_COMPLETE) { break; } Thread.sleep(50L); } - assertThat(consumer.getCurrentState(), equalTo(ConsumerStates.ShardConsumerState.SHUTDOWN_COMPLETE)); + assertThat(consumer.getCurrentState(), equalTo(KinesisConsumerStates.ShardConsumerState.SHUTDOWN_COMPLETE)); assertThat(processor.getShutdownReason(), is(equalTo(ShutdownReason.TERMINATE))); @@ -881,7 +881,7 @@ public class ShardConsumerTest { } /** - * Test method for {@link ShardConsumer#consumeShard()} that starts from initial position of type AT_TIMESTAMP. + * Test method for {@link KinesisShardConsumer#consumeShard()} that starts from initial position of type AT_TIMESTAMP. */ @Test public final void testConsumeShardWithInitialPositionAtTimestamp() throws Exception { @@ -938,8 +938,8 @@ public class ShardConsumerTest { any(IMetricsFactory.class), anyInt())) .thenReturn(getRecordsCache); - ShardConsumer consumer = - new ShardConsumer(shardInfo, + KinesisShardConsumer consumer = + new KinesisShardConsumer(shardInfo, streamConfig, checkpoint, processor, @@ -958,11 +958,11 @@ public class ShardConsumerTest { shardSyncer, shardSyncStrategy); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); consumer.consumeShard(); // check on parent shards Thread.sleep(50L); consumer.consumeShard(); // start initialization - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.INITIALIZING))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.INITIALIZING))); consumer.consumeShard(); // initialize Thread.sleep(50L); @@ -973,7 +973,7 @@ public class ShardConsumerTest { boolean newTaskSubmitted = consumer.consumeShard(); if (newTaskSubmitted) { LOG.debug("New processing task was submitted, call # " + i); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.PROCESSING))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.PROCESSING))); // CHECKSTYLE:IGNORE ModifiedControlVariable FOR NEXT 1 LINES i += maxRecords; } @@ -985,9 +985,9 @@ public class ShardConsumerTest { assertThat(processor.getShutdownReason(), nullValue()); consumer.beginShutdown(); Thread.sleep(50L); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.SHUTTING_DOWN))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.SHUTTING_DOWN))); consumer.beginShutdown(); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.SHUTDOWN_COMPLETE))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.SHUTDOWN_COMPLETE))); assertThat(processor.getShutdownReason(), is(equalTo(ShutdownReason.ZOMBIE))); executorService.shutdown(); @@ -1014,8 +1014,8 @@ public class ShardConsumerTest { callProcessRecordsForEmptyRecordList, skipCheckpointValidationValue, INITIAL_POSITION_LATEST); - ShardConsumer consumer = - new ShardConsumer(shardInfo, + KinesisShardConsumer consumer = + new KinesisShardConsumer(shardInfo, streamConfig, checkpoint, processor, @@ -1041,22 +1041,22 @@ public class ShardConsumerTest { when(checkpoint.getCheckpointObject(anyString())).thenReturn( new Checkpoint(checkpointSequenceNumber, pendingCheckpointSequenceNumber)); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); consumer.consumeShard(); // submit BlockOnParentShardTask Thread.sleep(50L); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.WAITING_ON_PARENT_SHARDS))); verify(processor, times(0)).initialize(any(InitializationInput.class)); consumer.consumeShard(); // submit InitializeTask Thread.sleep(50L); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.INITIALIZING))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.INITIALIZING))); verify(processor, times(1)).initialize(argThat( initializationInputMatcher(checkpointSequenceNumber, pendingCheckpointSequenceNumber))); verify(processor, times(1)).initialize(any(InitializationInput.class)); // no other calls with different args consumer.consumeShard(); Thread.sleep(50L); - assertThat(consumer.getCurrentState(), is(equalTo(ConsumerStates.ShardConsumerState.PROCESSING))); + assertThat(consumer.getCurrentState(), is(equalTo(KinesisConsumerStates.ShardConsumerState.PROCESSING))); } @Test @@ -1069,8 +1069,8 @@ public class ShardConsumerTest { callProcessRecordsForEmptyRecordList, skipCheckpointValidationValue, INITIAL_POSITION_LATEST); - ShardConsumer shardConsumer = - new ShardConsumer(shardInfo, + KinesisShardConsumer shardConsumer = + new KinesisShardConsumer(shardInfo, streamConfig, checkpoint, processor, @@ -1101,8 +1101,8 @@ public class ShardConsumerTest { callProcessRecordsForEmptyRecordList, skipCheckpointValidationValue, INITIAL_POSITION_LATEST); - ShardConsumer shardConsumer = - new ShardConsumer(shardInfo, + KinesisShardConsumer shardConsumer = + new KinesisShardConsumer(shardInfo, streamConfig, checkpoint, processor, @@ -1144,7 +1144,7 @@ public class ShardConsumerTest { skipCheckpointValidationValue, INITIAL_POSITION_LATEST); - ShardConsumer shardConsumer = new ShardConsumer( + KinesisShardConsumer shardConsumer = new KinesisShardConsumer( shardInfo, streamConfig, checkpoint, diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTaskTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTaskTest.java index 053a8bf7..e8870d66 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTaskTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShutdownTaskTest.java @@ -66,7 +66,7 @@ import com.amazonaws.services.kinesis.leases.interfaces.ILeaseManager; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; -import static com.amazonaws.services.kinesis.clientlibrary.lib.worker.ShutdownTask.RETRY_RANDOM_MAX_RANGE; +import static com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisShutdownTask.RETRY_RANDOM_MAX_RANGE; /** * @@ -139,7 +139,7 @@ public class ShutdownTaskTest { } /** - * Test method for {@link ShutdownTask#call()}. + * Test method for {@link KinesisShutdownTask#call()}. */ @Test public final void testCallWhenApplicationDoesNotCheckpoint() { @@ -148,7 +148,7 @@ public class ShutdownTaskTest { when(leaseCoordinator.getLeaseManager()).thenReturn(leaseManager); boolean cleanupLeasesOfCompletedShards = false; boolean ignoreUnexpectedChildShards = false; - ShutdownTask task = new ShutdownTask(defaultShardInfo, + KinesisShutdownTask task = new KinesisShutdownTask(defaultShardInfo, defaultRecordProcessor, checkpointer, ShutdownReason.TERMINATE, @@ -171,7 +171,7 @@ public class ShutdownTaskTest { } /** - * Test method for {@link ShutdownTask#call()}. + * Test method for {@link KinesisShutdownTask#call()}. */ @Test public final void testCallWhenCreatingLeaseThrows() throws Exception { @@ -183,7 +183,7 @@ public class ShutdownTaskTest { final String exceptionMessage = "InvalidStateException is thrown."; when(leaseManager.createLeaseIfNotExists(any(KinesisClientLease.class))).thenThrow(new InvalidStateException(exceptionMessage)); - ShutdownTask task = new ShutdownTask(defaultShardInfo, + KinesisShutdownTask task = new KinesisShutdownTask(defaultShardInfo, defaultRecordProcessor, checkpointer, ShutdownReason.TERMINATE, @@ -226,7 +226,7 @@ public class ShutdownTaskTest { // Make first 5 attempts with partial parent info in lease table for (int i = 0; i < 5; i++) { - ShutdownTask task = spy(new ShutdownTask(defaultShardInfo, + KinesisShutdownTask task = spy(new KinesisShutdownTask(defaultShardInfo, defaultRecordProcessor, checkpointer, ShutdownReason.TERMINATE, @@ -252,7 +252,7 @@ public class ShutdownTaskTest { } // Make next attempt with complete parent info in lease table - ShutdownTask task = spy(new ShutdownTask(defaultShardInfo, + KinesisShutdownTask task = spy(new KinesisShutdownTask(defaultShardInfo, defaultRecordProcessor, checkpointer, ShutdownReason.TERMINATE, @@ -290,7 +290,7 @@ public class ShutdownTaskTest { when(leaseManager.getLease("ShardId-1")).thenReturn(null, null, null, null, null, null, null, null, null, null, null); for (int i = 0; i < 10; i++) { - ShutdownTask task = spy(new ShutdownTask(defaultShardInfo, + KinesisShutdownTask task = spy(new KinesisShutdownTask(defaultShardInfo, defaultRecordProcessor, checkpointer, ShutdownReason.TERMINATE, @@ -315,7 +315,7 @@ public class ShutdownTaskTest { verify(defaultRecordProcessor, never()).shutdown(any(ShutdownInput.class)); } - ShutdownTask task = spy(new ShutdownTask(defaultShardInfo, + KinesisShutdownTask task = spy(new KinesisShutdownTask(defaultShardInfo, defaultRecordProcessor, checkpointer, ShutdownReason.TERMINATE, @@ -351,7 +351,7 @@ public class ShutdownTaskTest { boolean cleanupLeasesOfCompletedShards = false; boolean ignoreUnexpectedChildShards = false; - ShutdownTask task = new ShutdownTask(defaultShardInfo, + KinesisShutdownTask task = new KinesisShutdownTask(defaultShardInfo, defaultRecordProcessor, checkpointer, ShutdownReason.TERMINATE, @@ -385,7 +385,7 @@ public class ShutdownTaskTest { boolean cleanupLeasesOfCompletedShards = false; boolean ignoreUnexpectedChildShards = false; - ShutdownTask task = new ShutdownTask(shardInfo, + KinesisShutdownTask task = new KinesisShutdownTask(shardInfo, defaultRecordProcessor, checkpointer, ShutdownReason.TERMINATE, @@ -415,7 +415,7 @@ public class ShutdownTaskTest { boolean cleanupLeasesOfCompletedShards = false; boolean ignoreUnexpectedChildShards = false; - ShutdownTask task = new ShutdownTask(defaultShardInfo, + KinesisShutdownTask task = new KinesisShutdownTask(defaultShardInfo, defaultRecordProcessor, checkpointer, ShutdownReason.ZOMBIE, @@ -438,12 +438,12 @@ public class ShutdownTaskTest { } /** - * Test method for {@link ShutdownTask#getTaskType()}. + * Test method for {@link KinesisShutdownTask#getTaskType()}. */ @Test public final void testGetTaskType() { KinesisClientLibLeaseCoordinator leaseCoordinator = mock(KinesisClientLibLeaseCoordinator.class); - ShutdownTask task = new ShutdownTask(null, null, null, null, + KinesisShutdownTask task = new KinesisShutdownTask(null, null, null, null, null, null, false, false, leaseCoordinator, 0, getRecordsCache, shardSyncer, shardSyncStrategy, Collections.emptyList(), leaseCleanupManager); diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/WorkerTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/WorkerTest.java index 50a3aa9b..75d74186 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/WorkerTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/WorkerTest.java @@ -186,7 +186,7 @@ public class WorkerTest { @Mock private IRecordProcessor v2RecordProcessor; @Mock - private ShardConsumer shardConsumer; + private IShardConsumer shardConsumer; @Mock private Future taskFuture; @Mock @@ -297,13 +297,13 @@ public class WorkerTest { KinesisClientLibConfiguration.DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST, shardPrioritization); ShardInfo shardInfo = new ShardInfo(dummyKinesisShardId, testConcurrencyToken, null, ExtendedSequenceNumber.TRIM_HORIZON); - ShardConsumer consumer = worker.createOrGetShardConsumer(shardInfo, streamletFactory); + IShardConsumer consumer = worker.createOrGetShardConsumer(shardInfo, streamletFactory); Assert.assertNotNull(consumer); - ShardConsumer consumer2 = worker.createOrGetShardConsumer(shardInfo, streamletFactory); + IShardConsumer consumer2 = worker.createOrGetShardConsumer(shardInfo, streamletFactory); Assert.assertSame(consumer, consumer2); ShardInfo shardInfoWithSameShardIdButDifferentConcurrencyToken = new ShardInfo(dummyKinesisShardId, anotherConcurrencyToken, null, ExtendedSequenceNumber.TRIM_HORIZON); - ShardConsumer consumer3 = + IShardConsumer consumer3 = worker.createOrGetShardConsumer(shardInfoWithSameShardIdButDifferentConcurrencyToken, streamletFactory); Assert.assertNotNull(consumer3); Assert.assertNotSame(consumer3, consumer); @@ -419,10 +419,10 @@ public class WorkerTest { new ShardInfo(dummyKinesisShardId, anotherConcurrencyToken, null, ExtendedSequenceNumber.TRIM_HORIZON); ShardInfo shardInfo2 = new ShardInfo(anotherDummyKinesisShardId, concurrencyToken, null, ExtendedSequenceNumber.TRIM_HORIZON); - ShardConsumer consumerOfShardInfo1 = worker.createOrGetShardConsumer(shardInfo1, streamletFactory); - ShardConsumer consumerOfDuplicateOfShardInfo1ButWithAnotherConcurrencyToken = + IShardConsumer consumerOfShardInfo1 = worker.createOrGetShardConsumer(shardInfo1, streamletFactory); + IShardConsumer consumerOfDuplicateOfShardInfo1ButWithAnotherConcurrencyToken = worker.createOrGetShardConsumer(duplicateOfShardInfo1ButWithAnotherConcurrencyToken, streamletFactory); - ShardConsumer consumerOfShardInfo2 = worker.createOrGetShardConsumer(shardInfo2, streamletFactory); + IShardConsumer consumerOfShardInfo2 = worker.createOrGetShardConsumer(shardInfo2, streamletFactory); Set assignedShards = new HashSet(); assignedShards.add(shardInfo1); @@ -1219,11 +1219,11 @@ public class WorkerTest { false, shardPrioritization); - final Map shardInfoShardConsumerMap = worker.getShardInfoShardConsumerMap(); + final Map shardInfoShardConsumerMap = worker.getShardInfoShardConsumerMap(); final ShardInfo completedShardInfo = KinesisClientLibLeaseCoordinator.convertLeaseToAssignment(completedLease); - final ShardConsumer completedShardConsumer = mock(ShardConsumer.class); + final KinesisShardConsumer completedShardConsumer = mock(KinesisShardConsumer.class); shardInfoShardConsumerMap.put(completedShardInfo, completedShardConsumer); - when(completedShardConsumer.getCurrentState()).thenReturn(ConsumerStates.ShardConsumerState.SHUTDOWN_COMPLETE); + when(completedShardConsumer.getCurrentState()).thenReturn(KinesisConsumerStates.ShardConsumerState.SHUTDOWN_COMPLETE); Callable callable = worker.createWorkerShutdownCallable(); assertThat(worker.hasGracefulShutdownStarted(), equalTo(false)); @@ -1338,11 +1338,11 @@ public class WorkerTest { verify(executorService).submit(argThat(both(isA(MetricsCollectingTaskDecorator.class)) .and(TaskTypeMatcher.isOfType(TaskType.SHUTDOWN)).and(ReflectionFieldMatcher - .withField(ShutdownTask.class, "shardInfo", equalTo(shardInfo1))))); + .withField(KinesisShutdownTask.class, "shardInfo", equalTo(shardInfo1))))); verify(executorService, never()).submit(argThat(both(isA(MetricsCollectingTaskDecorator.class)) .and(TaskTypeMatcher.isOfType(TaskType.SHUTDOWN)).and(ReflectionFieldMatcher - .withField(ShutdownTask.class, "shardInfo", equalTo(shardInfo2))))); + .withField(KinesisShutdownTask.class, "shardInfo", equalTo(shardInfo2))))); } @@ -1451,11 +1451,11 @@ public class WorkerTest { verify(executorService, never()).submit(argThat(both(isA(MetricsCollectingTaskDecorator.class)) .and(TaskTypeMatcher.isOfType(TaskType.SHUTDOWN)).and(ReflectionFieldMatcher - .withField(ShutdownTask.class, "shardInfo", equalTo(shardInfo1))))); + .withField(KinesisShutdownTask.class, "shardInfo", equalTo(shardInfo1))))); verify(executorService, never()).submit(argThat(both(isA(MetricsCollectingTaskDecorator.class)) .and(TaskTypeMatcher.isOfType(TaskType.SHUTDOWN)).and(ReflectionFieldMatcher - .withField(ShutdownTask.class, "shardInfo", equalTo(shardInfo2))))); + .withField(KinesisShutdownTask.class, "shardInfo", equalTo(shardInfo2))))); @@ -2013,19 +2013,19 @@ public class WorkerTest { @Override protected boolean matchesSafely(MetricsCollectingTaskDecorator item, Description mismatchDescription) { return Condition.matched(item, mismatchDescription) - .and(new Condition.Step() { + .and(new Condition.Step() { @Override - public Condition apply(MetricsCollectingTaskDecorator value, + public Condition apply(MetricsCollectingTaskDecorator value, Description mismatch) { - if (!(value.getOther() instanceof ShutdownTask)) { + if (!(value.getOther() instanceof KinesisShutdownTask)) { mismatch.appendText("Wrapped task isn't a shutdown task"); return Condition.notMatched(); } - return Condition.matched((ShutdownTask) value.getOther(), mismatch); + return Condition.matched((KinesisShutdownTask) value.getOther(), mismatch); } - }).and(new Condition.Step() { + }).and(new Condition.Step() { @Override - public Condition apply(ShutdownTask value, Description mismatch) { + public Condition apply(KinesisShutdownTask value, Description mismatch) { return Condition.matched(value.getReason(), mismatch); } }).matching(matcher); From 367cede700193896f4b69bfedaa6f85363d319dd Mon Sep 17 00:00:00 2001 From: kdsjasonwng <111027619+kdsjasonwng@users.noreply.github.com> Date: Wed, 14 Dec 2022 11:07:54 -0800 Subject: [PATCH 33/48] Preparation for v1.14.9 (#1012) --- CHANGELOG.md | 6 +++++- README.md | 6 +++++- pom.xml | 2 +- .../lib/worker/KinesisClientLibConfiguration.java | 2 +- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43772079..205a7401 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ # Changelog -### Latest Release (1.14.8 - Feb 24, 2021) +### Latest Release (1.14.9 - Dec 14, 2022) +* [#995](https://github.com/awslabs/amazon-kinesis-client/commit/372f98b21a91487e36612d528c56765a44b0aa86) Every other change for DynamoDBStreamsKinesis Adapter Compatibility +* [#970](https://github.com/awslabs/amazon-kinesis-client/commit/251b331a2e0fd912b50f8b5a12d088bf0b3263b9) PeriodicShardSyncManager Changes Needed for DynamoDBStreamsKinesisAdapter + +### Release (1.14.8 - Feb 24, 2022) * [Bump log4j-core from 2.17.0 to 2.17.1](https://github.com/awslabs/amazon-kinesis-client/commit/94b138a9d9a502ee0f4f000bb0efd2766ebadc37) * [Bump protobuf-java from 3.19.1 to 3.19.4](https://github.com/awslabs/amazon-kinesis-client/commit/a809b12c43c57a3d6ad3827feb60e4322614259c) * [Bump maven-compiler-plugin from 3.8.1 to 3.10.0](https://github.com/awslabs/amazon-kinesis-client/commit/37b5d7b9a1ccad483469ef542a6a7237462b14f2) diff --git a/README.md b/README.md index 5da7a029..792d8b14 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,11 @@ To make it easier for developers to write record processors in other languages, ## Release Notes -### Latest Release (1.14.8 - Feb 24, 2021) +### Latest Release (1.14.9 - Dec 14, 2022) +* [#995](https://github.com/awslabs/amazon-kinesis-client/commit/372f98b21a91487e36612d528c56765a44b0aa86) Every other change for DynamoDBStreamsKinesis Adapter Compatibility +* [#970](https://github.com/awslabs/amazon-kinesis-client/commit/251b331a2e0fd912b50f8b5a12d088bf0b3263b9) PeriodicShardSyncManager Changes Needed for DynamoDBStreamsKinesisAdapter + +### Release (1.14.8 - Feb 24, 2022) * [Bump log4j-core from 2.17.0 to 2.17.1](https://github.com/awslabs/amazon-kinesis-client/commit/94b138a9d9a502ee0f4f000bb0efd2766ebadc37) * [Bump protobuf-java from 3.19.1 to 3.19.4](https://github.com/awslabs/amazon-kinesis-client/commit/a809b12c43c57a3d6ad3827feb60e4322614259c) * [Bump maven-compiler-plugin from 3.8.1 to 3.10.0](https://github.com/awslabs/amazon-kinesis-client/commit/37b5d7b9a1ccad483469ef542a6a7237462b14f2) diff --git a/pom.xml b/pom.xml index e87ede7d..85358f1e 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ amazon-kinesis-client jar Amazon Kinesis Client Library for Java - 1.14.8 + 1.14.9 The Amazon Kinesis Client Library for Java enables Java developers to easily consume and process data from Amazon Kinesis. diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java index 7aa2f885..acf0aa3e 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java @@ -147,7 +147,7 @@ public class KinesisClientLibConfiguration { /** * User agent set when Amazon Kinesis Client Library makes AWS requests. */ - public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.14.8"; + public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.14.9"; /** * KCL will validate client provided sequence numbers with a call to Amazon Kinesis before checkpointing for calls From c72d38bdc10d977ba048d76c8b862a2ca7c82d62 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 3 Feb 2023 12:42:38 -0500 Subject: [PATCH 34/48] Bump maven-javadoc-plugin from 3.3.1 to 3.4.1 (#978) Bumps [maven-javadoc-plugin](https://github.com/apache/maven-javadoc-plugin) from 3.3.1 to 3.4.1. - [Release notes](https://github.com/apache/maven-javadoc-plugin/releases) - [Commits](https://github.com/apache/maven-javadoc-plugin/compare/maven-javadoc-plugin-3.3.1...maven-javadoc-plugin-3.4.1) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-javadoc-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 85358f1e..fcbae8ae 100644 --- a/pom.xml +++ b/pom.xml @@ -263,7 +263,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.3.1 + 3.4.1 com.amazonaws.services.kinesis.producer.protobuf From d5324409eece8d5e9db929872fbc0eb59dfa7b8a Mon Sep 17 00:00:00 2001 From: ZeyuLi-AWS <125080976+ZeyuLi-AWS@users.noreply.github.com> Date: Thu, 16 Feb 2023 12:51:48 -0800 Subject: [PATCH 35/48] Updated the dependency to fix vulnerability issue (#1042) * updated the aws-java-sdk and google.protobuf version to fix vulnerabilities --- CHANGELOG.md | 7 ++++++- pom.xml | 6 +++--- .../lib/worker/KinesisClientLibConfiguration.java | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 205a7401..af32970e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ # Changelog +### Latest Release (1.14.10 - Feb 15, 2023) +* Updated aws-java-sdk from 1.12.130 to 1.12.406 +* Updated com.google.protobuf from 3.19.4 to 3.19.6 + * [Issue #1026](https://github.com/awslabs/amazon-kinesis-client/issues/1026) + * [PR #1042](https://github.com/awslabs/amazon-kinesis-client/pull/1042) -### Latest Release (1.14.9 - Dec 14, 2022) +### Release (1.14.9 - Dec 14, 2022) * [#995](https://github.com/awslabs/amazon-kinesis-client/commit/372f98b21a91487e36612d528c56765a44b0aa86) Every other change for DynamoDBStreamsKinesis Adapter Compatibility * [#970](https://github.com/awslabs/amazon-kinesis-client/commit/251b331a2e0fd912b50f8b5a12d088bf0b3263b9) PeriodicShardSyncManager Changes Needed for DynamoDBStreamsKinesisAdapter diff --git a/pom.xml b/pom.xml index fcbae8ae..38cabe7a 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ amazon-kinesis-client jar Amazon Kinesis Client Library for Java - 1.14.9 + 1.14.10 The Amazon Kinesis Client Library for Java enables Java developers to easily consume and process data from Amazon Kinesis. @@ -25,7 +25,7 @@ - 1.12.130 + 1.12.406 1.0.392 libsqlite4java ${project.build.directory}/test-lib @@ -60,7 +60,7 @@ com.google.protobuf protobuf-java - 3.19.4 + 3.19.6 org.apache.commons diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java index acf0aa3e..bf95586a 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java @@ -147,7 +147,7 @@ public class KinesisClientLibConfiguration { /** * User agent set when Amazon Kinesis Client Library makes AWS requests. */ - public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.14.9"; + public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.14.10"; /** * KCL will validate client provided sequence numbers with a call to Amazon Kinesis before checkpointing for calls From 43edc90c278b9b750fc3b1e0dc6cb1e568446885 Mon Sep 17 00:00:00 2001 From: ZeyuLi-AWS <125080976+ZeyuLi-AWS@users.noreply.github.com> Date: Tue, 21 Feb 2023 05:51:27 -0800 Subject: [PATCH 36/48] Add release notes 1.14.10 in README.md (#1049) --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 792d8b14..99d222a4 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,13 @@ To make it easier for developers to write record processors in other languages, ## Release Notes -### Latest Release (1.14.9 - Dec 14, 2022) +### Latest Release (1.14.10 - Feb 15, 2023) +* Updated aws-java-sdk from 1.12.130 to 1.12.406 +* Updated com.google.protobuf from 3.19.4 to 3.19.6 + * [Issue #1026](https://github.com/awslabs/amazon-kinesis-client/issues/1026) + * [PR #1042](https://github.com/awslabs/amazon-kinesis-client/pull/1042) + +### Release (1.14.9 - Dec 14, 2022) * [#995](https://github.com/awslabs/amazon-kinesis-client/commit/372f98b21a91487e36612d528c56765a44b0aa86) Every other change for DynamoDBStreamsKinesis Adapter Compatibility * [#970](https://github.com/awslabs/amazon-kinesis-client/commit/251b331a2e0fd912b50f8b5a12d088bf0b3263b9) PeriodicShardSyncManager Changes Needed for DynamoDBStreamsKinesisAdapter From ef251d4a84e46e4e4dcf99b9c054c6462d06a056 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 May 2023 14:24:07 -0700 Subject: [PATCH 37/48] Bump maven-compiler-plugin from 3.10.0 to 3.11.0 (#1055) Bumps [maven-compiler-plugin](https://github.com/apache/maven-compiler-plugin) from 3.10.0 to 3.11.0. - [Release notes](https://github.com/apache/maven-compiler-plugin/releases) - [Commits](https://github.com/apache/maven-compiler-plugin/compare/maven-compiler-plugin-3.10.0...maven-compiler-plugin-3.11.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-compiler-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 38cabe7a..8943f9b2 100644 --- a/pom.xml +++ b/pom.xml @@ -147,7 +147,7 @@ org.apache.maven.plugins maven-compiler-plugin - 3.10.0 + 3.11.0 1.8 1.8 From 9ab4206844344e39cd613d1698bb182d2d2b9527 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 May 2023 15:22:40 -0700 Subject: [PATCH 38/48] Bump aws-java-sdk.version from 1.12.406 to 1.12.408 (#1044) Bumps `aws-java-sdk.version` from 1.12.406 to 1.12.408. Updates `aws-java-sdk-core` from 1.12.406 to 1.12.408 - [Release notes](https://github.com/aws/aws-sdk-java/releases) - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.12.406...1.12.408) Updates `aws-java-sdk-dynamodb` from 1.12.406 to 1.12.408 - [Release notes](https://github.com/aws/aws-sdk-java/releases) - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.12.406...1.12.408) Updates `aws-java-sdk-kinesis` from 1.12.406 to 1.12.408 - [Release notes](https://github.com/aws/aws-sdk-java/releases) - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.12.406...1.12.408) Updates `aws-java-sdk-cloudwatch` from 1.12.406 to 1.12.408 - [Release notes](https://github.com/aws/aws-sdk-java/releases) - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.12.406...1.12.408) --- updated-dependencies: - dependency-name: com.amazonaws:aws-java-sdk-core dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: com.amazonaws:aws-java-sdk-dynamodb dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: com.amazonaws:aws-java-sdk-kinesis dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: com.amazonaws:aws-java-sdk-cloudwatch dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8943f9b2..074ae28e 100644 --- a/pom.xml +++ b/pom.xml @@ -25,7 +25,7 @@ - 1.12.406 + 1.12.408 1.0.392 libsqlite4java ${project.build.directory}/test-lib From 640780abfb846e31b947bf47d365e866fc927ef2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 May 2023 15:35:37 -0700 Subject: [PATCH 39/48] Bump nexus-staging-maven-plugin from 1.6.8 to 1.6.13 (#943) Bumps nexus-staging-maven-plugin from 1.6.8 to 1.6.13. --- updated-dependencies: - dependency-name: org.sonatype.plugins:nexus-staging-maven-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 074ae28e..2e60884f 100644 --- a/pom.xml +++ b/pom.xml @@ -336,7 +336,7 @@ org.sonatype.plugins nexus-staging-maven-plugin - 1.6.8 + 1.6.13 true sonatype-nexus-staging From 078a3ca2c457a38f8fdf8d523c89bac53d0c1e3a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 May 2023 13:48:33 -0700 Subject: [PATCH 40/48] Bump jackson-core from 2.13.0 to 2.15.0 (#1103) Bumps [jackson-core](https://github.com/FasterXML/jackson-core) from 2.13.0 to 2.15.0. - [Release notes](https://github.com/FasterXML/jackson-core/releases) - [Changelog](https://github.com/FasterXML/jackson-core/blob/jackson-core-2.15.0/release.properties) - [Commits](https://github.com/FasterXML/jackson-core/compare/jackson-core-2.13.0...jackson-core-2.15.0) --- updated-dependencies: - dependency-name: com.fasterxml.jackson.core:jackson-core dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2e60884f..cdc6be2a 100644 --- a/pom.xml +++ b/pom.xml @@ -118,7 +118,7 @@ com.fasterxml.jackson.core jackson-core - 2.13.0 + 2.15.0 From e6db97222618a12fb80b017171e630dfd04312d0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 17 May 2023 10:39:01 -0700 Subject: [PATCH 41/48] Bump log4j-core from 2.17.1 to 2.20.0 (#1054) * Bump log4j-core from 2.17.1 to 2.20.0 Bumps [log4j-core](https://github.com/apache/logging-log4j2) from 2.17.1 to 2.20.0. - [Release notes](https://github.com/apache/logging-log4j2/releases) - [Changelog](https://github.com/apache/logging-log4j2/blob/release-2.x/CHANGELOG.adoc) - [Commits](https://github.com/apache/logging-log4j2/compare/rel/2.17.1...rel/2.20.0) --- updated-dependencies: - dependency-name: org.apache.logging.log4j:log4j-core dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Add log4j-api 2.20.0 dependency to fix broken build by log4j-core bump to 2.20.0 --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: lucienlu-aws <132623944+lucienlu-aws@users.noreply.github.com> --- pom.xml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index cdc6be2a..46a169fa 100644 --- a/pom.xml +++ b/pom.xml @@ -111,7 +111,14 @@ org.apache.logging.log4j log4j-core - 2.17.1 + 2.20.0 + test + + + + org.apache.logging.log4j + log4j-api + 2.20.0 test From fe1a34f6e5ee1729e537f2bd0a3c6859c110d7a0 Mon Sep 17 00:00:00 2001 From: lucienlu-aws <132623944+lucienlu-aws@users.noreply.github.com> Date: Wed, 17 May 2023 16:41:28 -0700 Subject: [PATCH 42/48] Prevent HashRangesAreAlwaysComplete test from building invalid hierarchy trees (#1111) --- .../lib/worker/PeriodicShardSyncManagerTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PeriodicShardSyncManagerTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PeriodicShardSyncManagerTest.java index 779ba92f..240b8459 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PeriodicShardSyncManagerTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PeriodicShardSyncManagerTest.java @@ -50,6 +50,7 @@ public class PeriodicShardSyncManagerTest { private static final String WORKER_ID = "workerId"; public static final long LEASES_RECOVERY_AUDITOR_EXECUTION_FREQUENCY_MILLIS = 2 * 60 * 1000L; public static final int LEASES_RECOVERY_AUDITOR_INCONSISTENCY_CONFIDENCE_THRESHOLD = 3; + private static final int MAX_DEPTH_WITH_IN_PROGRESS_PARENTS = 1; /** Manager for PERIODIC shard sync strategy */ private PeriodicShardSyncManager periodicShardSyncManager; @@ -475,7 +476,7 @@ public class PeriodicShardSyncManagerTest { for (int i = 0; i < 1000; i++) { int maxInitialLeaseCount = 100; List leases = generateInitialLeases(maxInitialLeaseCount); - reshard(leases, 5, ReshardType.MERGE, maxInitialLeaseCount, true); + reshard(leases, MAX_DEPTH_WITH_IN_PROGRESS_PARENTS, ReshardType.MERGE, maxInitialLeaseCount, true); Collections.shuffle(leases); Assert.assertFalse(periodicShardSyncManager.hasHoleInLeases(leases).isPresent()); Assert.assertFalse(auditorPeriodicShardSyncManager.hasHoleInLeases(leases).isPresent()); @@ -487,7 +488,7 @@ public class PeriodicShardSyncManagerTest { for (int i = 0; i < 1000; i++) { int maxInitialLeaseCount = 100; List leases = generateInitialLeases(maxInitialLeaseCount); - reshard(leases, 5, ReshardType.ANY, maxInitialLeaseCount, true); + reshard(leases, MAX_DEPTH_WITH_IN_PROGRESS_PARENTS, ReshardType.ANY, maxInitialLeaseCount, true); Collections.shuffle(leases); Assert.assertFalse(periodicShardSyncManager.hasHoleInLeases(leases).isPresent()); Assert.assertFalse(auditorPeriodicShardSyncManager.hasHoleInLeases(leases).isPresent()); From db30ac956b0c91a81af64a1ec771157fbccc91e7 Mon Sep 17 00:00:00 2001 From: lucienlu-aws <132623944+lucienlu-aws@users.noreply.github.com> Date: Wed, 7 Jun 2023 13:55:51 -0700 Subject: [PATCH 43/48] Add support for Stream ARNs (#1108) Add support for referencing streams by streamARN --- .../worker/KinesisClientLibConfiguration.java | 128 ++++++++++++- .../clientlibrary/proxies/KinesisProxy.java | 16 ++ .../KinesisClientLibConfigurationTest.java | 168 +++++++++++++++--- .../proxies/KinesisProxyTest.java | 121 ++++++++++--- 4 files changed, 382 insertions(+), 51 deletions(-) diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java index bf95586a..19555cda 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java @@ -17,12 +17,14 @@ package com.amazonaws.services.kinesis.clientlibrary.lib.worker; import java.time.Duration; import java.util.Date; import java.util.Optional; +import java.util.regex.Pattern; import java.util.Set; import com.amazonaws.services.dynamodbv2.model.BillingMode; import org.apache.commons.lang3.Validate; import com.amazonaws.ClientConfiguration; +import com.amazonaws.arn.Arn; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.services.kinesis.metrics.impl.MetricsHelper; import com.amazonaws.services.kinesis.metrics.interfaces.IMetricsScope; @@ -233,11 +235,25 @@ public class KinesisClientLibConfiguration { */ public static final int DEFAULT_MAX_INITIALIZATION_ATTEMPTS = 20; + /** + * Pattern for a stream ARN. The valid format is + * {@code arn:aws:kinesis:::stream/} + * where {@code region} is the id representation of a {@link Region}. + */ + private static final Pattern STREAM_ARN_PATTERN = Pattern.compile( + "arn:aws[^:]*:kinesis:(?[-a-z0-9]+):(?[0-9]{12}):stream/(?.+)"); + @Getter private BillingMode billingMode; private String applicationName; private String tableName; private String streamName; + + /** + * Kinesis stream ARN + */ + @Getter + private Arn streamArn; private String kinesisEndpoint; private String dynamoDBEndpoint; private InitialPositionInStream initialPositionInStream; @@ -719,14 +735,105 @@ public class KinesisClientLibConfiguration { this.billingMode = billingMode; } + /** + * Duplicate constructor to support stream ARN's in place of stream names. + * + * @param applicationName Name of the Kinesis application + * By default the application name is included in the user agent string used to make AWS requests. This + * can assist with troubleshooting (e.g. distinguish requests made by separate applications). + * @param streamArn Kinesis stream ARN + * @param kinesisEndpoint Kinesis endpoint + * @param dynamoDBEndpoint DynamoDB endpoint + * @param initialPositionInStream One of LATEST or TRIM_HORIZON. The KinesisClientLibrary will start fetching + * records from that location in the stream when an application starts up for the first time and there + * are no checkpoints. If there are checkpoints, then we start from the checkpoint position. + * @param kinesisCredentialsProvider Provides credentials used to access Kinesis + * @param dynamoDBCredentialsProvider Provides credentials used to access DynamoDB + * @param cloudWatchCredentialsProvider Provides credentials used to access CloudWatch + * @param failoverTimeMillis Lease duration (leases not renewed within this period will be claimed by others) + * @param workerId Used to distinguish different workers/processes of a Kinesis application + * @param maxRecords Max records to read per Kinesis getRecords() call + * @param idleTimeBetweenReadsInMillis Idle time between calls to fetch data from Kinesis + * @param callProcessRecordsEvenForEmptyRecordList Call the IRecordProcessor::processRecords() API even if + * GetRecords returned an empty record list. + * @param parentShardPollIntervalMillis Wait for this long between polls to check if parent shards are done + * @param shardSyncIntervalMillis Time between tasks to sync leases and Kinesis shards + * @param cleanupTerminatedShardsBeforeExpiry Clean up shards we've finished processing (don't wait for expiration + * in Kinesis) + * @param kinesisClientConfig Client Configuration used by Kinesis client + * @param dynamoDBClientConfig Client Configuration used by DynamoDB client + * @param cloudWatchClientConfig Client Configuration used by CloudWatch client + * @param taskBackoffTimeMillis Backoff period when tasks encounter an exception + * @param metricsBufferTimeMillis Metrics are buffered for at most this long before publishing to CloudWatch + * @param metricsMaxQueueSize Max number of metrics to buffer before publishing to CloudWatch + * @param validateSequenceNumberBeforeCheckpointing whether KCL should validate client provided sequence numbers + * with a call to Amazon Kinesis before checkpointing for calls to + * {@link RecordProcessorCheckpointer#checkpoint(String)} + * @param regionName The region name for the service + * @param shutdownGraceMillis Time before gracefully shutdown forcefully terminates + * @param billingMode The DDB Billing mode to set for lease table creation. + * @param recordsFetcherFactory Factory to create the records fetcher to retrieve data from Kinesis for a given shard. + * @param leaseCleanupIntervalMillis Rate at which to run lease cleanup thread in + * {@link com.amazonaws.services.kinesis.leases.impl.LeaseCleanupManager} + * @param completedLeaseCleanupThresholdMillis Threshold in millis at which to check if there are any completed leases + * (leases for shards which have been closed as a result of a resharding operation) that need to be cleaned up. + * @param garbageLeaseCleanupThresholdMillis Threshold in millis at which to check if there are any garbage leases + * (leases for shards which no longer exist in the stream) that need to be cleaned up. + */ + public KinesisClientLibConfiguration(String applicationName, + Arn streamArn, + String kinesisEndpoint, + String dynamoDBEndpoint, + InitialPositionInStream initialPositionInStream, + AWSCredentialsProvider kinesisCredentialsProvider, + AWSCredentialsProvider dynamoDBCredentialsProvider, + AWSCredentialsProvider cloudWatchCredentialsProvider, + long failoverTimeMillis, + String workerId, + int maxRecords, + long idleTimeBetweenReadsInMillis, + boolean callProcessRecordsEvenForEmptyRecordList, + long parentShardPollIntervalMillis, + long shardSyncIntervalMillis, + boolean cleanupTerminatedShardsBeforeExpiry, + ClientConfiguration kinesisClientConfig, + ClientConfiguration dynamoDBClientConfig, + ClientConfiguration cloudWatchClientConfig, + long taskBackoffTimeMillis, + long metricsBufferTimeMillis, + int metricsMaxQueueSize, + boolean validateSequenceNumberBeforeCheckpointing, + String regionName, + long shutdownGraceMillis, + BillingMode billingMode, + RecordsFetcherFactory recordsFetcherFactory, + long leaseCleanupIntervalMillis, + long completedLeaseCleanupThresholdMillis, + long garbageLeaseCleanupThresholdMillis) { + this(applicationName, streamArn.getResource().getResource(), kinesisEndpoint, dynamoDBEndpoint, initialPositionInStream, kinesisCredentialsProvider, + dynamoDBCredentialsProvider, cloudWatchCredentialsProvider, failoverTimeMillis, workerId, maxRecords, idleTimeBetweenReadsInMillis, + callProcessRecordsEvenForEmptyRecordList, parentShardPollIntervalMillis, shardSyncIntervalMillis, cleanupTerminatedShardsBeforeExpiry, + kinesisClientConfig, dynamoDBClientConfig, cloudWatchClientConfig, taskBackoffTimeMillis, metricsBufferTimeMillis, + metricsMaxQueueSize, validateSequenceNumberBeforeCheckpointing, regionName, shutdownGraceMillis, billingMode, + recordsFetcherFactory, leaseCleanupIntervalMillis, completedLeaseCleanupThresholdMillis, garbageLeaseCleanupThresholdMillis); + checkIsValidStreamArn(streamArn); + this.streamArn = streamArn; + } + // Check if value is positive, otherwise throw an exception - private void checkIsValuePositive(String key, long value) { + private static void checkIsValuePositive(String key, long value) { if (value <= 0) { throw new IllegalArgumentException("Value of " + key + " should be positive, but current value is " + value); } } + private static void checkIsValidStreamArn(Arn streamArn) { + if (!STREAM_ARN_PATTERN.matcher(streamArn.toString()).matches()) { + throw new IllegalArgumentException("Invalid streamArn " + streamArn); + } + } + // Check if user agent in configuration is the default agent. // If so, replace it with application name plus KINESIS_CLIENT_LIB_USER_AGENT. // If not, append KINESIS_CLIENT_LIB_USER_AGENT to the end. @@ -1056,6 +1163,25 @@ public class KinesisClientLibConfiguration { return shutdownGraceMillis; } + /** + * @param streamName Kinesis stream name + * @return KinesisClientLibConfiguration + */ + public KinesisClientLibConfiguration withStreamName(String streamName) { + this.streamName = streamName; + return this; + } + + /** + * @param streamArn Kinesis stream ARN + * @return KinesisClientLibConfiguration + */ + public KinesisClientLibConfiguration withStreamArn(Arn streamArn) { + checkIsValidStreamArn(streamArn); + this.streamArn = streamArn; + return this; + } + /* // CHECKSTYLE:IGNORE HiddenFieldCheck FOR NEXT 190 LINES /** diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxy.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxy.java index c6b641bb..5673aeb9 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxy.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxy.java @@ -37,6 +37,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import com.amazonaws.arn.Arn; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.services.kinesis.AmazonKinesis; import com.amazonaws.services.kinesis.AmazonKinesisClient; @@ -94,6 +95,12 @@ public class KinesisProxy implements IKinesisProxyExtended { private final String streamName; + /** + * Stored as a string instead of an ARN to reduce repetitive null checks when passing in the stream ARN to + * the client requests, which accepts a String stream ARN parameter. + */ + private String streamArn; + private static final long DEFAULT_DESCRIBE_STREAM_BACKOFF_MILLIS = 1000L; private static final int DEFAULT_DESCRIBE_STREAM_RETRY_TIMES = 50; private final long describeStreamBackoffTimeInMillis; @@ -218,6 +225,8 @@ public class KinesisProxy implements IKinesisProxyExtended { config.getListShardsBackoffTimeInMillis(), config.getMaxListShardsRetryAttempts()); this.credentialsProvider = config.getKinesisCredentialsProvider(); + Arn arn = config.getStreamArn(); + this.streamArn = arn != null ? arn.toString() : null; } public KinesisProxy(final String streamName, @@ -253,6 +262,7 @@ public class KinesisProxy implements IKinesisProxyExtended { final GetRecordsRequest getRecordsRequest = new GetRecordsRequest(); getRecordsRequest.setRequestCredentials(credentialsProvider.getCredentials()); + getRecordsRequest.setStreamARN(streamArn); getRecordsRequest.setShardIterator(shardIterator); getRecordsRequest.setLimit(maxRecords); final GetRecordsResult response = client.getRecords(getRecordsRequest); @@ -270,6 +280,7 @@ public class KinesisProxy implements IKinesisProxyExtended { final DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest(); describeStreamRequest.setRequestCredentials(credentialsProvider.getCredentials()); describeStreamRequest.setStreamName(streamName); + describeStreamRequest.setStreamARN(streamArn); describeStreamRequest.setExclusiveStartShardId(startShardId); DescribeStreamResult response = null; @@ -314,6 +325,7 @@ public class KinesisProxy implements IKinesisProxyExtended { request.setRequestCredentials(credentialsProvider.getCredentials()); if (StringUtils.isEmpty(nextToken)) { request.setStreamName(streamName); + request.setStreamARN(streamArn); request.setShardFilter(shardFilter); } else { request.setNextToken(nextToken); @@ -567,6 +579,7 @@ public class KinesisProxy implements IKinesisProxyExtended { final GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest(); getShardIteratorRequest.setRequestCredentials(credentialsProvider.getCredentials()); getShardIteratorRequest.setStreamName(streamName); + getShardIteratorRequest.setStreamARN(streamArn); getShardIteratorRequest.setShardId(shardId); getShardIteratorRequest.setShardIteratorType(iteratorType); getShardIteratorRequest.setStartingSequenceNumber(sequenceNumber); @@ -583,6 +596,7 @@ public class KinesisProxy implements IKinesisProxyExtended { final GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest(); getShardIteratorRequest.setRequestCredentials(credentialsProvider.getCredentials()); getShardIteratorRequest.setStreamName(streamName); + getShardIteratorRequest.setStreamARN(streamArn); getShardIteratorRequest.setShardId(shardId); getShardIteratorRequest.setShardIteratorType(iteratorType); getShardIteratorRequest.setStartingSequenceNumber(null); @@ -599,6 +613,7 @@ public class KinesisProxy implements IKinesisProxyExtended { final GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest(); getShardIteratorRequest.setRequestCredentials(credentialsProvider.getCredentials()); getShardIteratorRequest.setStreamName(streamName); + getShardIteratorRequest.setStreamARN(streamArn); getShardIteratorRequest.setShardId(shardId); getShardIteratorRequest.setShardIteratorType(ShardIteratorType.AT_TIMESTAMP); getShardIteratorRequest.setStartingSequenceNumber(null); @@ -618,6 +633,7 @@ public class KinesisProxy implements IKinesisProxyExtended { final PutRecordRequest putRecordRequest = new PutRecordRequest(); putRecordRequest.setRequestCredentials(credentialsProvider.getCredentials()); putRecordRequest.setStreamName(streamName); + putRecordRequest.setStreamARN(streamArn); putRecordRequest.setSequenceNumberForOrdering(exclusiveMinimumSequenceNumber); putRecordRequest.setExplicitHashKey(explicitHashKey); putRecordRequest.setPartitionKey(partitionKey); diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfigurationTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfigurationTest.java index f2b5a460..aeb2e0c9 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfigurationTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfigurationTest.java @@ -20,13 +20,16 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertFalse; import static org.junit.Assert.fail; +import java.util.Arrays; import java.util.Date; +import java.util.List; import com.amazonaws.services.dynamodbv2.model.BillingMode; import org.junit.Test; import org.mockito.Mockito; import com.amazonaws.ClientConfiguration; +import com.amazonaws.arn.Arn; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.regions.Region; import com.amazonaws.regions.RegionUtils; @@ -34,6 +37,7 @@ import com.amazonaws.services.cloudwatch.AmazonCloudWatchClient; import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; import com.amazonaws.services.kinesis.AmazonKinesisClient; import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorFactory; +import com.amazonaws.services.kinesis.clientlibrary.lib.worker.SimpleRecordsFetcherFactory; import com.amazonaws.services.kinesis.metrics.interfaces.MetricsLevel; import com.google.common.collect.ImmutableSet; @@ -46,8 +50,28 @@ public class KinesisClientLibConfigurationTest { private static final long TEST_VALUE_LONG = 1000L; private static final int TEST_VALUE_INT = 1000; private static final int PARAMETER_COUNT = 6; + private static final String ACCOUNT_ID = "123456789012"; private static final String TEST_STRING = "TestString"; + private static final Arn TEST_ARN = Arn.builder() + .withPartition("aws") + .withService("kinesis") + .withRegion("us-east-1") + .withAccountId(ACCOUNT_ID) + .withResource("stream/" + TEST_STRING) + .build(); + + /** + * Invalid steamARN due to invalid service. This is a sample used for testing. + * @see KinesisClientLibConfigurationTest#testWithInvalidStreamArnsThrowsException() for more examples + */ + private static final Arn INVALID_TEST_ARN = Arn.builder() + .withPartition("aws") + .withService("dynamodb") + .withRegion("us-east-1") + .withAccountId(ACCOUNT_ID) + .withResource("stream/" + TEST_STRING) + .build(); private static final String ALTER_STRING = "AlterString"; // We don't want any of these tests to run checkpoint validation @@ -62,32 +86,11 @@ public class KinesisClientLibConfigurationTest { new KinesisClientLibConfiguration(TEST_STRING, TEST_STRING, null, TEST_STRING); // Test constructor with all valid arguments. - config = - new KinesisClientLibConfiguration(TEST_STRING, - TEST_STRING, - TEST_STRING, - TEST_STRING, - InitialPositionInStream.LATEST, - null, - null, - null, - TEST_VALUE_LONG, - TEST_STRING, - TEST_VALUE_INT, - TEST_VALUE_LONG, - false, - TEST_VALUE_LONG, - TEST_VALUE_LONG, - true, - new ClientConfiguration(), - new ClientConfiguration(), - new ClientConfiguration(), - TEST_VALUE_LONG, - TEST_VALUE_LONG, - TEST_VALUE_INT, - skipCheckpointValidationValue, - null, - TEST_VALUE_LONG, BillingMode.PROVISIONED); + config = buildKinesisClientLibConfiguration(TEST_STRING); + + // Test constructor with streamArn with all valid arguments. + config = buildKinesisClientLibConfiguration(TEST_ARN); + Assert.assertEquals(config.getStreamName(), TEST_STRING); } @Test @@ -168,6 +171,12 @@ public class KinesisClientLibConfigurationTest { } intValues[i] = TEST_VALUE_INT; } + // Test constructor with invalid streamArn + try { + config = buildKinesisClientLibConfiguration(INVALID_TEST_ARN); + } catch(IllegalArgumentException e) { + System.out.println(e.getMessage()); + } Assert.assertTrue("KCLConfiguration should return null when using negative arguments", config == null); } @@ -370,4 +379,111 @@ public class KinesisClientLibConfigurationTest { config = config.withIgnoreUnexpectedChildShards(true); assertTrue(config.shouldIgnoreUnexpectedChildShards()); } + + @Test + public void testWithValidStreamArnsSucceed() { + List validArnList = Arrays.asList( + "arn:aws:kinesis:us-east-1:123456789012:stream/123stream-name123", + "arn:aws-china:kinesis:us-east-2:123456789012:stream/stream-name", + "arn:aws-us-gov:kinesis:us-east-2:123456789012:stream/stream-name" + ); + + KinesisClientLibConfiguration config = + new KinesisClientLibConfiguration("TestApplication", "TestStream", null, "TestWorker"); + + for (final String arn : validArnList) { + config.withStreamArn(Arn.fromString(arn)); + } + } + + @Test + public void testWithInvalidStreamArnsThrowsException() { + List invalidArnList = Arrays.asList( + "arn:abc:kinesis:us-east-1:123456789012:stream/stream-name", //invalid partition + "arn:aws:dynamnodb:us-east-1:123456789012:stream/stream-name", // Kinesis ARN, but with a non-Kinesis service + "arn:aws:kinesis::123456789012:stream/stream-name", // missing region + "arn:aws:kinesis:us-east-1::stream/stream-name", // missing account id + "arn:aws:kinesis:us-east-1:123456789:stream/stream-name", // account id not 12 digits + "arn:aws:kinesis:us-east-1:123456789abc:stream/stream-name", // 12char alphanumeric account id + "arn:aws:kinesis:us-east-1:123456789012:table/table-name", // incorrect resource type + "arn:aws:dynamodb:us-east-1:123456789012:table/myDynamoDBTable" // valid arn but not a stream + ); + + KinesisClientLibConfiguration config = + new KinesisClientLibConfiguration("TestApplication", "TestStream", null, "TestWorker"); + + for (final String arnString : invalidArnList) { + Arn arn = Arn.fromString(arnString); + try { + config.withStreamArn(arn); + fail("Arn " + arn + " should have thrown an IllegalArgumentException"); + } catch (IllegalArgumentException e) { + // expected + } + } + } + + private KinesisClientLibConfiguration buildKinesisClientLibConfiguration(Arn streamArn) { + return new KinesisClientLibConfiguration(TEST_STRING, + streamArn, + TEST_STRING, + TEST_STRING, + InitialPositionInStream.LATEST, + null, + null, + null, + TEST_VALUE_LONG, + TEST_STRING, + TEST_VALUE_INT, + TEST_VALUE_LONG, + false, + TEST_VALUE_LONG, + TEST_VALUE_LONG, + true, + new ClientConfiguration(), + new ClientConfiguration(), + new ClientConfiguration(), + TEST_VALUE_LONG, + TEST_VALUE_LONG, + TEST_VALUE_INT, + skipCheckpointValidationValue, + null, + TEST_VALUE_LONG, BillingMode.PROVISIONED, + new SimpleRecordsFetcherFactory(), + TEST_VALUE_LONG, + TEST_VALUE_LONG, + TEST_VALUE_LONG); + } + + private KinesisClientLibConfiguration buildKinesisClientLibConfiguration(String streamName) { + return new KinesisClientLibConfiguration(TEST_STRING, + streamName, + TEST_STRING, + TEST_STRING, + InitialPositionInStream.LATEST, + null, + null, + null, + TEST_VALUE_LONG, + TEST_STRING, + TEST_VALUE_INT, + TEST_VALUE_LONG, + false, + TEST_VALUE_LONG, + TEST_VALUE_LONG, + true, + new ClientConfiguration(), + new ClientConfiguration(), + new ClientConfiguration(), + TEST_VALUE_LONG, + TEST_VALUE_LONG, + TEST_VALUE_INT, + skipCheckpointValidationValue, + null, + TEST_VALUE_LONG, BillingMode.PROVISIONED, + new SimpleRecordsFetcherFactory(), + TEST_VALUE_LONG, + TEST_VALUE_LONG, + TEST_VALUE_LONG); + } } diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxyTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxyTest.java index 76671176..a603db1d 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxyTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxyTest.java @@ -48,6 +48,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import java.util.stream.Collectors; +import lombok.Builder; import org.apache.commons.lang3.StringUtils; import org.hamcrest.Description; import org.hamcrest.TypeSafeDiagnosingMatcher; @@ -59,6 +60,7 @@ import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; import com.amazonaws.AmazonServiceException; +import com.amazonaws.arn.Arn; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.services.dynamodbv2.streamsadapter.AmazonDynamoDBStreamsAdapterClient; import com.amazonaws.services.dynamodbv2.streamsadapter.AmazonDynamoDBStreamsAdapterClientChild; @@ -77,11 +79,17 @@ import com.amazonaws.services.kinesis.model.ShardIteratorType; import com.amazonaws.services.kinesis.model.StreamDescription; import com.amazonaws.services.kinesis.model.StreamStatus; -import lombok.AllArgsConstructor; - @RunWith(MockitoJUnitRunner.class) public class KinesisProxyTest { private static final String TEST_STRING = "TestString"; + private static final String ACCOUNT_ID = "123456789012"; + private static final Arn TEST_ARN = Arn.builder() + .withPartition("aws") + .withService("kinesis") + .withRegion("us-east-1") + .withAccountId(ACCOUNT_ID) + .withResource("stream/" + TEST_STRING) + .build(); private static final long DESCRIBE_STREAM_BACKOFF_TIME = 10L; private static final long LIST_SHARDS_BACKOFF_TIME = 10L; private static final int DESCRIBE_STREAM_RETRY_TIMES = 3; @@ -132,6 +140,7 @@ public class KinesisProxyTest { public void setUpTest() { // Set up kinesis ddbProxy when(config.getStreamName()).thenReturn(TEST_STRING); + when(config.getStreamArn()).thenReturn(TEST_ARN); when(config.getListShardsBackoffTimeInMillis()).thenReturn(LIST_SHARDS_BACKOFF_TIME); when(config.getMaxListShardsRetryAttempts()).thenReturn(LIST_SHARDS_RETRY_TIMES); when(config.getKinesisCredentialsProvider()).thenReturn(mockCredentialsProvider); @@ -163,7 +172,8 @@ public class KinesisProxyTest { // Second call describeStream returning response with rest shards. DescribeStreamResult responseWithMoreData = createGetStreamInfoResponse(shards.subList(0, 2), true); DescribeStreamResult responseFinal = createGetStreamInfoResponse(shards.subList(2, shards.size()), false); - doReturn(responseWithMoreData).when(mockDDBStreamClient).describeStream(argThat(new IsRequestWithStartShardId(null))); + IsRequestWithStartShardId requestMatcher = IsRequestWithStartShardId.builder().streamName(TEST_STRING).build(); + doReturn(responseWithMoreData).when(mockDDBStreamClient).describeStream(argThat(requestMatcher)); doReturn(responseFinal).when(mockDDBStreamClient) .describeStream(argThat(new OldIsRequestWithStartShardId(shards.get(1).getShardId()))); @@ -310,7 +320,8 @@ public class KinesisProxyTest { public void testGetShardListWithDDBChildClient() { DescribeStreamResult responseWithMoreData = createGetStreamInfoResponse(shards.subList(0, 2), true); DescribeStreamResult responseFinal = createGetStreamInfoResponse(shards.subList(2, shards.size()), false); - doReturn(responseWithMoreData).when(mockDDBChildClient).describeStream(argThat(new IsRequestWithStartShardId(null))); + IsRequestWithStartShardId requestMatcher = IsRequestWithStartShardId.builder().streamName(TEST_STRING).build(); + doReturn(responseWithMoreData).when(mockDDBChildClient).describeStream(argThat(requestMatcher)); doReturn(responseFinal).when(mockDDBChildClient) .describeStream(argThat(new OldIsRequestWithStartShardId(shards.get(1).getShardId()))); @@ -498,37 +509,61 @@ public class KinesisProxyTest { return response; } - private IsRequestWithStartShardId describeWithoutShardId() { - return describeWithShardId(null); - } - private IsRequestWithStartShardId describeWithShardId(String shardId) { - return new IsRequestWithStartShardId(shardId); + return IsRequestWithStartShardId.builder() + .streamName(TEST_STRING) + .streamArn(TEST_ARN) + .shardId(shardId) + .build(); } + @Builder private static class IsRequestWithStartShardId extends TypeSafeDiagnosingMatcher { + private final String streamName; + private final Arn streamArn; private final String shardId; - public IsRequestWithStartShardId(String shardId) { - this.shardId = shardId; - } - @Override protected boolean matchesSafely(DescribeStreamRequest item, Description mismatchDescription) { + boolean matches = true; + if (streamName == null) { + if (item.getStreamName() != null) { + mismatchDescription.appendText("Expected streamName of null, but was ") + .appendValue(item.getStreamName()); + matches = false; + } + } else if (!streamName.equals(item.getStreamName())) { + mismatchDescription.appendValue(streamName).appendText(" doesn't match expected ") + .appendValue(item.getStreamName()); + matches = false; + } + + if (streamArn == null) { + if (item.getStreamARN() != null) { + mismatchDescription.appendText("Expected streamArn of null, but was ") + .appendValue(item.getStreamARN()); + matches = false; + } + } else if (!streamArn.equals(Arn.fromString(item.getStreamARN()))) { + mismatchDescription.appendValue(streamArn).appendText(" doesn't match expected ") + .appendValue(item.getStreamARN()); + matches = false; + } + if (shardId == null) { if (item.getExclusiveStartShardId() != null) { mismatchDescription.appendText("Expected starting shard id of null, but was ") .appendValue(item.getExclusiveStartShardId()); - return false; + matches = false; } } else if (!shardId.equals(item.getExclusiveStartShardId())) { mismatchDescription.appendValue(shardId).appendText(" doesn't match expected ") .appendValue(item.getExclusiveStartShardId()); - return false; + matches = false; } - return true; + return matches; } @Override @@ -557,49 +592,87 @@ public class KinesisProxyTest { } private static ListShardsRequestMatcher initialListShardsRequestMatcher() { - return new ListShardsRequestMatcher(null, null); + return ListShardsRequestMatcher.builder() + .streamName(TEST_STRING) + .streamArn(TEST_ARN) + .build(); } private static ListShardsRequestMatcher listShardsNextToken(final String nextToken) { - return new ListShardsRequestMatcher(null, nextToken); + return ListShardsRequestMatcher.builder() + .nextToken(nextToken) + .build(); } - @AllArgsConstructor + @Builder private static class ListShardsRequestMatcher extends TypeSafeDiagnosingMatcher { + private final String streamName; + private final Arn streamArn; private final String shardId; private final String nextToken; @Override protected boolean matchesSafely(final ListShardsRequest listShardsRequest, final Description description) { + boolean matches = true; + if (streamName == null) { + if (StringUtils.isNotEmpty(listShardsRequest.getStreamName())) { + description.appendText("Expected streamName to be null, but was ") + .appendValue(listShardsRequest.getStreamName()); + matches = false; + } + } else { + if (!streamName.equals(listShardsRequest.getStreamName())) { + description.appendText("Expected streamName: ").appendValue(streamName) + .appendText(" doesn't match actual streamName: ") + .appendValue(listShardsRequest.getStreamName()); + matches = false; + } + } + + if (streamArn == null) { + if (StringUtils.isNotEmpty(listShardsRequest.getStreamARN())) { + description.appendText("Expected streamArn to be null, but was ") + .appendValue(listShardsRequest.getStreamARN()); + matches = false; + } + } else { + if (!streamArn.equals(Arn.fromString(listShardsRequest.getStreamARN()))) { + description.appendText("Expected streamArn: ").appendValue(streamArn) + .appendText(" doesn't match actual streamArn: ") + .appendValue(listShardsRequest.getStreamARN()); + matches = false; + } + } + if (shardId == null) { if (StringUtils.isNotEmpty(listShardsRequest.getExclusiveStartShardId())) { description.appendText("Expected ExclusiveStartShardId to be null, but was ") .appendValue(listShardsRequest.getExclusiveStartShardId()); - return false; + matches = false; } } else { if (!shardId.equals(listShardsRequest.getExclusiveStartShardId())) { description.appendText("Expected shardId: ").appendValue(shardId) .appendText(" doesn't match actual shardId: ") .appendValue(listShardsRequest.getExclusiveStartShardId()); - return false; + matches = false; } } if (StringUtils.isNotEmpty(listShardsRequest.getNextToken())) { if (StringUtils.isNotEmpty(listShardsRequest.getStreamName()) || StringUtils.isNotEmpty(listShardsRequest.getExclusiveStartShardId())) { - return false; + matches = false; } if (!listShardsRequest.getNextToken().equals(nextToken)) { description.appendText("Found nextToken: ").appendValue(listShardsRequest.getNextToken()) .appendText(" when it was supposed to be null."); - return false; + matches = false; } } else { return nextToken == null; } - return true; + return matches; } @Override From b011206f7bb96004892e76a72df2dc0bc3d17fc8 Mon Sep 17 00:00:00 2001 From: lucienlu-aws <132623944+lucienlu-aws@users.noreply.github.com> Date: Thu, 8 Jun 2023 14:11:42 -0700 Subject: [PATCH 44/48] Preparation for v1.15.0 (#1135) * Prepare changes to release v1.15.0 --- CHANGELOG.md | 11 ++++++++++- README.md | 11 ++++++++++- pom.xml | 2 +- .../lib/worker/KinesisClientLibConfiguration.java | 2 +- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index af32970e..c93b0ead 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog -### Latest Release (1.14.10 - Feb 15, 2023) +### Latest Release (1.15.0 - Jun 8, 2023) +* **[#1108](https://github.com/awslabs/amazon-kinesis-client/pull/1108) Add support for Stream ARNs** +* [#1111](https://github.com/awslabs/amazon-kinesis-client/pull/1111) More consistent testing behavior with HashRangesAreAlwaysComplete +* [#1054](https://github.com/awslabs/amazon-kinesis-client/pull/1054) Upgrade log4j-core from 2.17.1 to 2.20.0 +* [#1103](https://github.com/awslabs/amazon-kinesis-client/pull/1103) Upgrade jackson-core from 2.13.0 to 2.15.0 +* [#943](https://github.com/awslabs/amazon-kinesis-client/pull/943) Upgrade nexus-staging-maven-plugin from 1.6.8 to 1.6.13 +* [#1044](https://github.com/awslabs/amazon-kinesis-client/pull/1044) Upgrade aws-java-sdk.version from 1.12.406 to 1.12.408 +* [#1055](https://github.com/awslabs/amazon-kinesis-client/pull/1055) Upgrade maven-compiler-plugin from 3.10.0 to 3.11.0 + +### Release (1.14.10 - Feb 15, 2023) * Updated aws-java-sdk from 1.12.130 to 1.12.406 * Updated com.google.protobuf from 3.19.4 to 3.19.6 * [Issue #1026](https://github.com/awslabs/amazon-kinesis-client/issues/1026) diff --git a/README.md b/README.md index 99d222a4..263f61c0 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,16 @@ To make it easier for developers to write record processors in other languages, ## Release Notes -### Latest Release (1.14.10 - Feb 15, 2023) +### Latest Release (1.15.0 - Jun 8, 2023) +* **[#1108](https://github.com/awslabs/amazon-kinesis-client/pull/1108) Add support for Stream ARNs** +* [#1111](https://github.com/awslabs/amazon-kinesis-client/pull/1111) More consistent testing behavior with HashRangesAreAlwaysComplete +* [#1054](https://github.com/awslabs/amazon-kinesis-client/pull/1054) Upgrade log4j-core from 2.17.1 to 2.20.0 +* [#1103](https://github.com/awslabs/amazon-kinesis-client/pull/1103) Upgrade jackson-core from 2.13.0 to 2.15.0 +* [#943](https://github.com/awslabs/amazon-kinesis-client/pull/943) Upgrade nexus-staging-maven-plugin from 1.6.8 to 1.6.13 +* [#1044](https://github.com/awslabs/amazon-kinesis-client/pull/1044) Upgrade aws-java-sdk.version from 1.12.406 to 1.12.408 +* [#1055](https://github.com/awslabs/amazon-kinesis-client/pull/1055) Upgrade maven-compiler-plugin from 3.10.0 to 3.11.0 + +### Release (1.14.10 - Feb 15, 2023) * Updated aws-java-sdk from 1.12.130 to 1.12.406 * Updated com.google.protobuf from 3.19.4 to 3.19.6 * [Issue #1026](https://github.com/awslabs/amazon-kinesis-client/issues/1026) diff --git a/pom.xml b/pom.xml index 46a169fa..d00dc58c 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ amazon-kinesis-client jar Amazon Kinesis Client Library for Java - 1.14.10 + 1.15.0 The Amazon Kinesis Client Library for Java enables Java developers to easily consume and process data from Amazon Kinesis. diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java index 19555cda..4b1d894e 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java @@ -149,7 +149,7 @@ public class KinesisClientLibConfiguration { /** * User agent set when Amazon Kinesis Client Library makes AWS requests. */ - public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.14.10"; + public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.15.0"; /** * KCL will validate client provided sequence numbers with a call to Amazon Kinesis before checkpointing for calls From 1e6506cf97a39433db7e4fb72224739bde41fec3 Mon Sep 17 00:00:00 2001 From: mmankika-aws <134537845+mmankika-aws@users.noreply.github.com> Date: Wed, 21 Jun 2023 13:24:23 -0700 Subject: [PATCH 45/48] Adding github actions workflow for 1.x (#1146) --- .github/workflows/maven.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/workflows/maven.yml diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml new file mode 100644 index 00000000..47165a3f --- /dev/null +++ b/.github/workflows/maven.yml @@ -0,0 +1,32 @@ +# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-maven + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Java CI with Maven + +on: + push: + branches: + - 'v1.x' + pull_request: + branches: + - 'v1.x' + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up JDK 8 + uses: actions/setup-java@v3 + with: + java-version: '8' + distribution: 'corretto' + - name: Build with Maven + run: mvn -B package --file pom.xml -DskipITs \ No newline at end of file From e731486b7fad3c1c4ed060811f1016114ddb9302 Mon Sep 17 00:00:00 2001 From: Brendan Lynch <134438333+brendan-p-lynch@users.noreply.github.com> Date: Wed, 11 Oct 2023 14:54:49 -0700 Subject: [PATCH 46/48] Updated Guava version, added backoff for ShardSync Integ test, and reintroduced SNAPSHOT version (#1214) * Updated guava version to 32.1.1 and added backoff logic for ShardSync Integration test * Added back snapshot version classifier --------- Co-authored-by: Brendan Lynch --- pom.xml | 4 ++-- .../lib/worker/ShardSyncTaskIntegrationTest.java | 11 +++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index d00dc58c..f1f96381 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ amazon-kinesis-client jar Amazon Kinesis Client Library for Java - 1.15.0 + 1.15.1-SNAPSHOT The Amazon Kinesis Client Library for Java enables Java developers to easily consume and process data from Amazon Kinesis. @@ -55,7 +55,7 @@ com.google.guava guava - 31.0.1-jre + 32.1.1-jre com.google.protobuf diff --git a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTaskIntegrationTest.java b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTaskIntegrationTest.java index a4afa053..3df67220 100644 --- a/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTaskIntegrationTest.java +++ b/src/test/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTaskIntegrationTest.java @@ -42,6 +42,7 @@ import com.amazonaws.services.kinesis.leases.impl.KinesisClientLease; import com.amazonaws.services.kinesis.leases.impl.KinesisClientLeaseManager; import com.amazonaws.services.kinesis.leases.interfaces.IKinesisClientLeaseManager; import com.amazonaws.services.kinesis.model.StreamStatus; +import com.amazonaws.services.kinesis.model.LimitExceededException; import static junit.framework.TestCase.fail; @@ -58,6 +59,8 @@ public class ShardSyncTaskIntegrationTest { private IKinesisProxy kinesisProxy; private final KinesisShardSyncer shardSyncer = new KinesisShardSyncer(new KinesisLeaseCleanupValidator()); + private static final int retryBackoffMillis = 1000; + /** * @throws java.lang.Exception */ @@ -71,9 +74,13 @@ public class ShardSyncTaskIntegrationTest { } catch (AmazonServiceException ase) { } - StreamStatus status; + StreamStatus status = null; do { - status = StreamStatus.fromValue(kinesis.describeStream(STREAM_NAME).getStreamDescription().getStreamStatus()); + try { + status = StreamStatus.fromValue(kinesis.describeStream(STREAM_NAME).getStreamDescription().getStreamStatus()); + } catch (LimitExceededException e) { + Thread.sleep(retryBackoffMillis + (long) (Math.random() * 100)); + } } while (status != StreamStatus.ACTIVE); } From 1236d65f78de8d76106eb8f981c433dc3770f2c2 Mon Sep 17 00:00:00 2001 From: vincentvilo-aws <142546855+vincentvilo-aws@users.noreply.github.com> Date: Mon, 5 Feb 2024 10:46:41 -0800 Subject: [PATCH 47/48] Preparation for v1.15.1 release (#1252) * Updated aws-java-sdk from 1.12.406 to 1.12.647 * Prepare for v1.15.1 release * update CHANGELOG.md with changes for 1.15.1 * add 1.15.1 changes to README * include Guava update in changelog * Update release notes with concise descriptions --- CHANGELOG.md | 7 ++++++- README.md | 7 ++++++- pom.xml | 4 ++-- .../lib/worker/KinesisClientLibConfiguration.java | 2 +- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c93b0ead..34c2b575 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog -### Latest Release (1.15.0 - Jun 8, 2023) +### Latest Release (1.15.1 - Feb 5, 2023) +* [#1214](https://github.com/awslabs/amazon-kinesis-client/pull/1214) Added backoff logic for ShardSyncTaskIntegrationTest +* [#1214](https://github.com/awslabs/amazon-kinesis-client/pull/1214) Upgrade Guava version from 31.0.1 to 32.1.1 +* [#1252](https://github.com/awslabs/amazon-kinesis-client/pull/1252) Upgrade aws-java-sdk from 1.12.406 to 1.12.647 + +### Release (1.15.0 - Jun 8, 2023) * **[#1108](https://github.com/awslabs/amazon-kinesis-client/pull/1108) Add support for Stream ARNs** * [#1111](https://github.com/awslabs/amazon-kinesis-client/pull/1111) More consistent testing behavior with HashRangesAreAlwaysComplete * [#1054](https://github.com/awslabs/amazon-kinesis-client/pull/1054) Upgrade log4j-core from 2.17.1 to 2.20.0 diff --git a/README.md b/README.md index 263f61c0..c1db93e6 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,12 @@ To make it easier for developers to write record processors in other languages, ## Release Notes -### Latest Release (1.15.0 - Jun 8, 2023) +### Latest Release (1.15.1 - Feb 5, 2023) +* [#1214](https://github.com/awslabs/amazon-kinesis-client/pull/1214) Added backoff logic for ShardSyncTaskIntegrationTest +* [#1214](https://github.com/awslabs/amazon-kinesis-client/pull/1214) Upgrade Guava version from 31.0.1 to 32.1.1 +* [#1252](https://github.com/awslabs/amazon-kinesis-client/pull/1252) Upgrade aws-java-sdk from 1.12.406 to 1.12.647 + +### Release (1.15.0 - Jun 8, 2023) * **[#1108](https://github.com/awslabs/amazon-kinesis-client/pull/1108) Add support for Stream ARNs** * [#1111](https://github.com/awslabs/amazon-kinesis-client/pull/1111) More consistent testing behavior with HashRangesAreAlwaysComplete * [#1054](https://github.com/awslabs/amazon-kinesis-client/pull/1054) Upgrade log4j-core from 2.17.1 to 2.20.0 diff --git a/pom.xml b/pom.xml index f1f96381..f0e14e1e 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ amazon-kinesis-client jar Amazon Kinesis Client Library for Java - 1.15.1-SNAPSHOT + 1.15.1 The Amazon Kinesis Client Library for Java enables Java developers to easily consume and process data from Amazon Kinesis. @@ -25,7 +25,7 @@ - 1.12.408 + 1.12.647 1.0.392 libsqlite4java ${project.build.directory}/test-lib diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java index 4b1d894e..ffaa6e63 100644 --- a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java +++ b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java @@ -149,7 +149,7 @@ public class KinesisClientLibConfiguration { /** * User agent set when Amazon Kinesis Client Library makes AWS requests. */ - public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.15.0"; + public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.15.1"; /** * KCL will validate client provided sequence numbers with a call to Amazon Kinesis before checkpointing for calls From 2ab263f2734f1de1311710fa254e231a9bd8dbc9 Mon Sep 17 00:00:00 2001 From: vincentvilo-aws <142546855+vincentvilo-aws@users.noreply.github.com> Date: Tue, 6 Feb 2024 10:59:10 -0800 Subject: [PATCH 48/48] Reintroduce -SNAPSHOT version classifier (#1254) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f0e14e1e..ba7c207e 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ amazon-kinesis-client jar Amazon Kinesis Client Library for Java - 1.15.1 + 1.15.1-SNAPSHOT The Amazon Kinesis Client Library for Java enables Java developers to easily consume and process data from Amazon Kinesis.