luciano/amazon-kinesis-client
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #804 from kevioke/init-pos-timestamp

Browse source 
Allow InitialPositionInStreamExtended to be specified in properties file
This commit is contained in:
Avinash Chowdary 2021-06-01 14:58:13 -07:00 committed by GitHub
commit 4e9e86ec76
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 63 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/MultiLangDaemonConfiguration.java
View file

@ -19,6 +19,7 @@ import java.lang.reflect.InvocationTargetException;
import java.net.URI;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
@ -27,6 +28,7 @@ import java.util.UUID;
import java.util.function.Function;
import org.apache.commons.beanutils.BeanUtilsBean;
import org.apache.commons.beanutils.ConvertUtils;
import org.apache.commons.beanutils.ConvertUtilsBean;
import org.apache.commons.beanutils.Converter;
import org.apache.commons.beanutils.converters.ArrayConverter;
@ -197,6 +199,14 @@ public class MultiLangDaemonConfiguration {
this.utilsBean = utilsBean;
this.convertUtilsBean = convertUtilsBean;
convertUtilsBean.register(new Converter() {
@Override
public <T> T convert(Class<T> type, Object value) {
Date date = new Date(Long.parseLong(value.toString()) * 1000L);
return type.cast(InitialPositionInStreamExtended.newInitialPositionAtTimestamp(date));
}
}, InitialPositionInStreamExtended.class);
convertUtilsBean.register(new Converter() {
@Override
public <T> T convert(Class<T> type, Object value) {

59
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/KinesisClientLibConfiguratorTest.java
View file

@ -27,15 +27,15 @@ import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.net.URI;
import java.util.Arrays;
import java.util.Date;
import java.util.HashSet;
import java.util.Optional;
import java.util.Set;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import org.apache.commons.lang3.StringUtils;
import org.junit.Ignore;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.junit.Rule;
import org.junit.Test;
@ -45,11 +45,8 @@ import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.kinesis.common.InitialPositionInStream;
import software.amazon.kinesis.coordinator.KinesisClientLibConfiguration;
import software.amazon.kinesis.metrics.MetricsLevel;
import software.amazon.kinesis.processor.ShardRecordProcessorFactory;
@ -93,6 +90,47 @@ public class KinesisClientLibConfiguratorTest {
assertEquals(config.getShardSyncIntervalMillis(), 500);
}
@Test
public void testWithInitialPositionInStreamExtended() {
long epochTimeInSeconds = 1617406032;
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "applicationName = app",
"streamName = 123", "AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"initialPositionInStreamExtended = " + epochTimeInSeconds}, '\n'));
assertEquals(config.getInitialPositionInStreamExtended().getTimestamp(), new Date(epochTimeInSeconds * 1000L));
assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.AT_TIMESTAMP);
}
@Test
public void testInvalidInitialPositionInStream() {
// AT_TIMESTAMP cannot be used as initialPositionInStream. If a user wants to specify AT_TIMESTAMP,
// they must specify the time with initialPositionInStreamExtended.
try {
getConfiguration(StringUtils.join(new String[] { "applicationName = app",
"streamName = 123", "AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"initialPositionInStream = AT_TIMESTAMP"}, '\n'));
fail("Should have thrown when initialPositionInStream is set to AT_TIMESTAMP");
} catch (Exception e) {
Throwable rootCause = ExceptionUtils.getRootCause(e);
assertTrue(rootCause instanceof IllegalArgumentException);
}
}
@Test
public void testInvalidInitialPositionInStreamExtended() {
// initialPositionInStreamExtended takes a long value indicating seconds since epoch. If a non-long
// value is provided, the constructor should throw an IllegalArgumentException exception.
try {
getConfiguration(StringUtils.join(new String[] { "applicationName = app",
"streamName = 123", "AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"initialPositionInStreamExtended = null"}, '\n'));
fail("Should have thrown when initialPositionInStreamExtended is set to null");
} catch (Exception e) {
Throwable rootCause = ExceptionUtils.getRootCause(e);
assertTrue(rootCause instanceof IllegalArgumentException);
}
}
@Test
public void testWithUnsupportedClientConfigurationVariables() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
@ -159,7 +197,7 @@ public class KinesisClientLibConfiguratorTest {
}
@Test
public void testWithInitialPositionInStreamVariables() {
public void testWithInitialPositionInStreamTrimHorizon() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a",
"applicationName = b", "AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 123",
"initialPositionInStream = TriM_Horizon" }, '\n'));
@ -167,6 +205,15 @@ public class KinesisClientLibConfiguratorTest {
assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.TRIM_HORIZON);
}
@Test
public void testWithInitialPositionInStreamLatest() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a",
"applicationName = b", "AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 123",
"initialPositionInStream = LateSt" }, '\n'));
assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.LATEST);
}
@Test
public void testSkippingNonKCLVariables() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a",