luciano/amazon-kinesis-client
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add functionality to retry an InvalidArgumentException (#1270)

Browse source 
* add functionality to retry an InvalidArgumentException with a new iterator

* include DEFAULT_MAX_RECORDS value in IllegalArgumentException messages
This commit is contained in:
vincentvilo-aws 2024-03-08 13:37:25 -08:00 committed by GitHub
parent 63e0fe7537
commit 1280325c20
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 37 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/coordinator/KinesisClientLibConfiguration.java
View file

@ -977,6 +977,10 @@ public class KinesisClientLibConfiguration {
*/
public KinesisClientLibConfiguration withMaxRecords(int maxRecords) {
checkIsValuePositive("MaxRecords", (long) maxRecords);
if (maxRecords > DEFAULT_MAX_RECORDS) {
throw new IllegalArgumentException(
"maxRecords must be less than or equal to " + DEFAULT_MAX_RECORDS + " but current value is " + maxRecords);
}
this.maxRecords = maxRecords;
return this;
}

12
amazon-kinesis-client/src/main/java/software/amazon/kinesis/retrieval/polling/PollingConfig.java
View file

@ -42,6 +42,8 @@ public class PollingConfig implements RetrievalSpecificConfig {
public static final Duration DEFAULT_REQUEST_TIMEOUT = Duration.ofSeconds(30);
public static final int DEFAULT_MAX_RECORDS = 10000;
/**
* Configurable functional interface to override the existing DataFetcher.
*/
@ -73,7 +75,7 @@ public class PollingConfig implements RetrievalSpecificConfig {
* Default value: 10000
* </p>
*/
private int maxRecords = 10000;
private int maxRecords = DEFAULT_MAX_RECORDS;
/**
* @param streamName Name of Kinesis stream.
@ -144,6 +146,14 @@ public class PollingConfig implements RetrievalSpecificConfig {
return this;
}
public void maxRecords(int maxRecords) {
if (maxRecords > DEFAULT_MAX_RECORDS) {
throw new IllegalArgumentException(
"maxRecords must be less than or equal to " + DEFAULT_MAX_RECORDS + " but current value is " + maxRecords());
}
this.maxRecords = maxRecords;
}
/**
* The maximum time to wait for a future request from Kinesis to complete
*/

5
amazon-kinesis-client/src/main/java/software/amazon/kinesis/retrieval/polling/PrefetchRecordsPublisher.java
View file

@ -42,6 +42,7 @@ import software.amazon.awssdk.core.exception.SdkException;
import software.amazon.awssdk.services.cloudwatch.model.StandardUnit;
import software.amazon.awssdk.services.kinesis.model.ExpiredIteratorException;
import software.amazon.awssdk.services.kinesis.model.GetRecordsResponse;
import software.amazon.awssdk.services.kinesis.model.InvalidArgumentException;
import software.amazon.awssdk.services.kinesis.model.ProvisionedThroughputExceededException;
import software.amazon.kinesis.annotations.KinesisClientInternalApi;
import software.amazon.kinesis.common.InitialPositionInStreamExtended;
@ -102,7 +103,6 @@ public class PrefetchRecordsPublisher implements RecordsPublisher {
private final PublisherSession publisherSession;
private final ReentrantReadWriteLock resetLock = new ReentrantReadWriteLock();
private boolean wasReset = false;
private Instant lastEventDeliveryTime = Instant.EPOCH;
private final RequestDetails lastSuccessfulRequestDetails = new RequestDetails();
@ -512,6 +512,9 @@ public class PrefetchRecordsPublisher implements RecordsPublisher {
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
log.info("{} : Thread was interrupted, indicating shutdown was called on the cache.", streamAndShardId);
} catch (InvalidArgumentException e) {
log.info("{} : records threw InvalidArgumentException - iterator will be refreshed before retrying", streamAndShardId, e);
publisherSession.dataFetcher().restartIterator();
} catch (ExpiredIteratorException e) {
log.info("{} : records threw ExpiredIteratorException - restarting"
+ " after greatest seqNum passed to customer", streamAndShardId, e);

5
amazon-kinesis-client/src/test/java/software/amazon/kinesis/retrieval/polling/PollingConfigTest.java
View file

@ -44,4 +44,9 @@ public class PollingConfigTest {
config.validateState(true);
}
@Test(expected = IllegalArgumentException.class)
public void testInvalidRecordLimit() {
config.maxRecords(PollingConfig.DEFAULT_MAX_RECORDS + 1);
}
}

13
amazon-kinesis-client/src/test/java/software/amazon/kinesis/retrieval/polling/PrefetchRecordsPublisherTest.java
View file

@ -82,6 +82,7 @@ import software.amazon.awssdk.core.exception.SdkException;
import software.amazon.awssdk.services.kinesis.model.ChildShard;
import software.amazon.awssdk.services.kinesis.model.ExpiredIteratorException;
import software.amazon.awssdk.services.kinesis.model.GetRecordsResponse;
import software.amazon.awssdk.services.kinesis.model.InvalidArgumentException;
import software.amazon.awssdk.services.kinesis.model.Record;
import software.amazon.kinesis.common.InitialPositionInStreamExtended;
import software.amazon.kinesis.leases.ShardObjectHelper;
@ -451,6 +452,18 @@ public class PrefetchRecordsPublisherTest {
assertEquals(records.processRecordsInput().millisBehindLatest(), response.millisBehindLatest());
}
@Test
public void testInvalidArgumentExceptionIsRetried() {
when(getRecordsRetrievalStrategy.getRecords(MAX_RECORDS_PER_CALL))
.thenThrow(InvalidArgumentException.builder().build())
.thenReturn(getRecordsResponse);
getRecordsCache.start(sequenceNumber, initialPosition);
blockUntilConditionSatisfied(() -> getRecordsCache.getPublisherSession().prefetchRecordsQueue().size() == MAX_SIZE, 300);
verify(dataFetcher, times(1)).restartIterator();
}
@Test(timeout = 10000L)
public void testNoDeadlockOnFullQueue() {
//