luciano/amazon-kinesis-client
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #6 from ethkatnic/multilang-naming-convention-change

Browse source 
Replace AWSCredentialProvider with AwsCredentialProvider
This commit is contained in:
ethkatnic 2024-09-23 13:04:56 -07:00 committed by GitHub
commit fa361ca693
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 103 additions and 56 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MultiLangDaemon.java
View file

@ -63,8 +63,8 @@ import software.amazon.kinesis.coordinator.Scheduler;
* # Users can change the credentials provider the KCL will use to retrieve credentials.
* # The DefaultCredentialsProvider checks several other providers, which is
* # described here:
* # https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/auth/credentials/DefaultCredentialsProvider.html
* AWSCredentialsProvider = DefaultCredentialsProvider
* # https://sdk.amazonaws.com/java/api/2.0.0-preview-11/software/amazon/awssdk/auth/credentials/DefaultCredentialsProvider.html
* AwsCredentialsProvider = DefaultCredentialsProvider
* </pre>
*/
@Slf4j

17
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/KinesisClientLibConfigurator.java
View file

@ -28,7 +28,7 @@ import software.amazon.kinesis.common.StreamIdentifier;
/**
* KinesisClientLibConfigurator constructs a KinesisClientLibConfiguration from java properties file. The following
* three properties must be provided. 1) "applicationName" 2) "streamName" 3) "AWSCredentialsProvider"
* three properties must be provided. 1) "applicationName" 2) "streamName" 3) "AwsCredentialsProvider"
* KinesisClientLibConfigurator will help to automatically assign the value of "workerId" if this property is not
* provided. In the specified properties file, any properties, which matches the variable name in
* KinesisClientLibConfiguration and has a corresponding "with{variableName}" setter method, will be read in, and its
@ -62,7 +62,7 @@ public class KinesisClientLibConfigurator {
properties.entrySet().forEach(e -> {
try {
log.info("Processing (key={}, value={})", e.getKey(), e.getValue());
utilsBean.setProperty(configuration, (String) e.getKey(), e.getValue());
utilsBean.setProperty(configuration, processKey((String) e.getKey()), e.getValue());
} catch (IllegalAccessException | InvocationTargetException ex) {
throw new RuntimeException(ex);
}
@ -110,4 +110,17 @@ public class KinesisClientLibConfigurator {
}
return getConfiguration(properties);
}
/**
* Processes a configuration key to normalize AWS credentials provider naming. Necessary to conform to
* autogenerated setters.
* @param key the config param key
* @return case-configured param key name
*/
String processKey(String key) {
if (key.toLowerCase().startsWith("awscredentialsprovider")) {
key = key.replaceAll("(?i)awscredentialsprovider", "awsCredentialsProvider");
}
return key;
}
}

6
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/MultiLangDaemonConfiguration.java
View file

@ -195,19 +195,19 @@ public class MultiLangDaemonConfiguration {
private final BuilderDynaBean kinesisCredentialsProvider;
public void setAWSCredentialsProvider(String providerString) {
public void setAwsCredentialsProvider(String providerString) {
kinesisCredentialsProvider.set("", providerString);
}
private final BuilderDynaBean dynamoDBCredentialsProvider;
public void setAWSCredentialsProviderDynamoDB(String providerString) {
public void setAwsCredentialsProviderDynamoDB(String providerString) {
dynamoDBCredentialsProvider.set("", providerString);
}
private final BuilderDynaBean cloudWatchCredentialsProvider;
public void setAWSCredentialsProviderCloudWatch(String providerString) {
public void setAwsCredentialsProviderCloudWatch(String providerString) {
cloudWatchCredentialsProvider.set("", providerString);
}

4
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MultiLangDaemonConfigTest.java
View file

@ -65,7 +65,7 @@ public class MultiLangDaemonConfigTest {
String properties = String.format(
"executableName = %s\n"
+ "applicationName = %s\n"
+ "AWSCredentialsProvider = DefaultCredentialsProvider\n"
+ "AwsCredentialsProvider = DefaultCredentialsProvider\n"
+ "processingLanguage = malbolge\n"
+ "regionName = %s\n",
EXE, APPLICATION_NAME, "us-east-1");
@ -182,7 +182,7 @@ public class MultiLangDaemonConfigTest {
@Test
public void testPropertyValidation() {
String propertiesNoExecutableName = "applicationName = testApp \n" + "streamName = fakeStream \n"
+ "AWSCredentialsProvider = DefaultCredentialsProvider\n" + "processingLanguage = malbolge";
+ "AwsCredentialsProvider = DefaultCredentialsProvider\n" + "processingLanguage = malbolge";
ClassLoader classLoader = Mockito.mock(ClassLoader.class);
Mockito.doReturn(new ByteArrayInputStream(propertiesNoExecutableName.getBytes()))

12
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoderTest.java → amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/AwsCredentialsProviderPropertyValueDecoderTest.java
View file

@ -34,7 +34,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
public class AWSCredentialsProviderPropertyValueDecoderTest {
public class AwsCredentialsProviderPropertyValueDecoderTest {
private static final String TEST_ACCESS_KEY_ID = "123";
private static final String TEST_SECRET_KEY = "456";
@ -45,13 +45,13 @@ public class AWSCredentialsProviderPropertyValueDecoderTest {
private final AwsCredentialsProviderPropertyValueDecoder decoder = new AwsCredentialsProviderPropertyValueDecoder();
@ToString
private static class AWSCredentialsMatcher extends TypeSafeDiagnosingMatcher<AwsCredentialsProvider> {
private static class AwsCredentialsMatcher extends TypeSafeDiagnosingMatcher<AwsCredentialsProvider> {
private final Matcher<String> akidMatcher;
private final Matcher<String> secretMatcher;
private final Matcher<Class<?>> classMatcher;
public AWSCredentialsMatcher(String akid, String secret) {
public AwsCredentialsMatcher(String akid, String secret) {
this.akidMatcher = equalTo(akid);
this.secretMatcher = equalTo(secret);
this.classMatcher = instanceOf(AwsCredentialsProviderChain.class);
@ -81,13 +81,13 @@ public class AWSCredentialsProviderPropertyValueDecoderTest {
@Override
public void describeTo(Description description) {
description
.appendText("An AWSCredentialsProvider that provides an AWSCredential matching: ")
.appendText("An AwsCredentialsProvider that provides an AwsCredential matching: ")
.appendList("(", ", ", ")", Arrays.asList(classMatcher, akidMatcher, secretMatcher));
}
}
private static AWSCredentialsMatcher hasCredentials(String akid, String secret) {
return new AWSCredentialsMatcher(akid, secret);
private static AwsCredentialsMatcher hasCredentials(String akid, String secret) {
return new AwsCredentialsMatcher(akid, secret);
}
@Test

90
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/KinesisClientLibConfiguratorTest.java
View file

@ -59,7 +59,7 @@ public class KinesisClientLibConfiguratorTest {
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = " + credentialName1,
"AwsCredentialsProvider = " + credentialName1,
"workerId = 123"
},
'\n'));
@ -76,7 +76,7 @@ public class KinesisClientLibConfiguratorTest {
new String[] {
"applicationName = app",
"streamName = 123",
"AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"AwsCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"workerId = 123",
"failoverTimeMillis = 100",
"shardSyncIntervalMillis = 500"
@ -97,7 +97,7 @@ public class KinesisClientLibConfiguratorTest {
new String[] {
"applicationName = app",
"streamName = 123",
"AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"AwsCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"initialPositionInStreamExtended = " + epochTimeInSeconds
},
'\n'));
@ -115,7 +115,7 @@ public class KinesisClientLibConfiguratorTest {
new String[] {
"applicationName = app",
"streamName = 123",
"AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"AwsCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"initialPositionInStream = AT_TIMESTAMP"
},
'\n'));
@ -135,7 +135,7 @@ public class KinesisClientLibConfiguratorTest {
new String[] {
"applicationName = app",
"streamName = 123",
"AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"AwsCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"initialPositionInStreamExtended = null"
},
'\n'));
@ -150,7 +150,7 @@ public class KinesisClientLibConfiguratorTest {
public void testWithUnsupportedClientConfigurationVariables() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
new String[] {
"AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"AwsCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"workerId = id",
"kinesisClientConfig = {}",
"streamName = stream",
@ -169,7 +169,7 @@ public class KinesisClientLibConfiguratorTest {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
new String[] {
"streamName = kinesis",
"AWSCredentialsProvider = " + credentialName2 + ", " + credentialName1,
"AwsCredentialsProvider = " + credentialName2 + ", " + credentialName1,
"workerId = w123",
"maxRecords = 10",
"metricsMaxQueueSize = 20",
@ -194,7 +194,7 @@ public class KinesisClientLibConfiguratorTest {
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = ABCD, " + credentialName1,
"AwsCredentialsProvider = ABCD, " + credentialName1,
"workerId = 0",
"cleanupLeasesUponShardCompletion = false",
"validateSequenceNumberBeforeCheckpointing = true"
@ -214,7 +214,7 @@ public class KinesisClientLibConfiguratorTest {
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = ABCD," + credentialName1,
"AwsCredentialsProvider = ABCD," + credentialName1,
"workerId = 1",
"kinesisEndpoint = https://kinesis",
"metricsLevel = SUMMARY"
@ -232,7 +232,7 @@ public class KinesisClientLibConfiguratorTest {
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = ABCD," + credentialName1,
"AwsCredentialsProvider = ABCD," + credentialName1,
"workerId = 1",
"metricsEnabledDimensions = ShardId, WorkerIdentifier"
},
@ -252,7 +252,7 @@ public class KinesisClientLibConfiguratorTest {
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = ABCD," + credentialName1,
"AwsCredentialsProvider = ABCD," + credentialName1,
"workerId = 123",
"initialPositionInStream = TriM_Horizon"
},
@ -267,7 +267,7 @@ public class KinesisClientLibConfiguratorTest {
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = ABCD," + credentialName1,
"AwsCredentialsProvider = ABCD," + credentialName1,
"workerId = 123",
"initialPositionInStream = LateSt"
},
@ -282,7 +282,7 @@ public class KinesisClientLibConfiguratorTest {
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = ABCD," + credentialName1,
"AwsCredentialsProvider = ABCD," + credentialName1,
"workerId = 123",
"initialPositionInStream = TriM_Horizon",
"abc = 1"
@ -301,7 +301,7 @@ public class KinesisClientLibConfiguratorTest {
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = ABCD," + credentialName1,
"AwsCredentialsProvider = ABCD," + credentialName1,
"workerId = 123",
"initialPositionInStream = TriM_Horizon",
"maxGetRecordsThreadPool = 1"
@ -317,7 +317,7 @@ public class KinesisClientLibConfiguratorTest {
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = ABCD," + credentialName1,
"AwsCredentialsProvider = ABCD," + credentialName1,
"workerId = 123",
"initialPositionInStream = TriM_Horizon",
"maxGetRecordsThreadPool = 0",
@ -333,7 +333,7 @@ public class KinesisClientLibConfiguratorTest {
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = " + credentialName1,
"AwsCredentialsProvider = " + credentialName1,
"workerId = 123",
"failoverTimeMillis = 100nf"
},
@ -347,7 +347,7 @@ public class KinesisClientLibConfiguratorTest {
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = " + credentialName1,
"AwsCredentialsProvider = " + credentialName1,
"workerId = 123",
"failoverTimeMillis = -12"
},
@ -379,7 +379,7 @@ public class KinesisClientLibConfiguratorTest {
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = " + credentialName1,
"AwsCredentialsProvider = " + credentialName1,
"failoverTimeMillis = 100",
"shardSyncIntervalMillis = 500"
},
@ -396,7 +396,7 @@ public class KinesisClientLibConfiguratorTest {
String test = StringUtils.join(
new String[] {
"applicationName = b",
"AWSCredentialsProvider = " + credentialName1,
"AwsCredentialsProvider = " + credentialName1,
"workerId = 123",
"failoverTimeMillis = 100"
},
@ -409,7 +409,7 @@ public class KinesisClientLibConfiguratorTest {
String test = StringUtils.join(
new String[] {
"applicationName = b",
"AWSCredentialsProvider = " + credentialName1,
"AwsCredentialsProvider = " + credentialName1,
"workerId = 123",
"failoverTimeMillis = 100",
"streamName = ",
@ -424,7 +424,7 @@ public class KinesisClientLibConfiguratorTest {
String test = StringUtils.join(
new String[] {
"streamName = a",
"AWSCredentialsProvider = " + credentialName1,
"AwsCredentialsProvider = " + credentialName1,
"workerId = 123",
"failoverTimeMillis = 100"
},
@ -433,12 +433,12 @@ public class KinesisClientLibConfiguratorTest {
}
@Test
public void testWithAWSCredentialsFailed() {
public void testWithAwsCredentialsFailed() {
String test = StringUtils.join(
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = " + credentialName2,
"AwsCredentialsProvider = " + credentialName2,
"failoverTimeMillis = 100",
"shardSyncIntervalMillis = 500"
},
@ -456,16 +456,44 @@ public class KinesisClientLibConfiguratorTest {
}
}
@Test
public void testProcessKeyWithExpectedCasing() {
String key = "AwsCredentialsProvider";
String result = configurator.processKey(key);
assertEquals("awsCredentialsProvider", result);
}
@Test
public void testProcessKeyWithOldCasing() {
String key = "AWSCredentialsProvider";
String result = configurator.processKey(key);
assertEquals("awsCredentialsProvider", result);
}
@Test
public void testProcessKeyWithMixedCasing() {
String key = "AwScReDeNtIaLsPrOvIdEr";
String result = configurator.processKey(key);
assertEquals("awsCredentialsProvider", result);
}
@Test
public void testProcessKeyWithSuffix() {
String key = "awscredentialsproviderDynamoDB";
String result = configurator.processKey(key);
assertEquals("awsCredentialsProviderDynamoDB", result);
}
// TODO: fix this test
@Test
public void testWithDifferentAWSCredentialsForDynamoDBAndCloudWatch() {
public void testWithDifferentAwsCredentialsForDynamoDBAndCloudWatch() {
String test = StringUtils.join(
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = " + credentialNameKinesis,
"AWSCredentialsProviderDynamoDB = " + credentialNameDynamoDB,
"AWSCredentialsProviderCloudWatch = " + credentialNameCloudWatch,
"AwsCredentialsProvider = " + credentialNameKinesis,
"AwsCredentialsProviderDynamoDB = " + credentialNameDynamoDB,
"AwsCredentialsProviderCloudWatch = " + credentialNameCloudWatch,
"failoverTimeMillis = 100",
"shardSyncIntervalMillis = 500"
},
@ -486,14 +514,14 @@ public class KinesisClientLibConfiguratorTest {
// TODO: fix this test
@Test
public void testWithDifferentAWSCredentialsForDynamoDBAndCloudWatchFailed() {
public void testWithDifferentAwsCredentialsForDynamoDBAndCloudWatchFailed() {
String test = StringUtils.join(
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = " + credentialNameKinesis,
"AWSCredentialsProviderDynamoDB = " + credentialName2,
"AWSCredentialsProviderCloudWatch = " + credentialName2,
"AwsCredentialsProvider = " + credentialNameKinesis,
"AwsCredentialsProviderDynamoDB = " + credentialName2,
"AwsCredentialsProviderCloudWatch = " + credentialName2,
"failoverTimeMillis = 100",
"shardSyncIntervalMillis = 500"
},

4
amazon-kinesis-client-multilang/src/test/resources/multilang.properties
View file

@ -17,10 +17,12 @@ streamName = kclpysample
applicationName = MultiLangTest
# Users can change the credentials provider the KCL will use to retrieve credentials.
# Expected key name (case-sensitive):
# AwsCredentialsProvider / AwsCredentialsProviderDynamoDB / AwsCredentialsProviderCloudWatch
# The DefaultCredentialsProvider checks several other providers, which is
# described here:
# https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/auth/credentials/DefaultCredentialsProvider.html
AWSCredentialsProvider = DefaultCredentialsProvider
AwsCredentialsProvider = DefaultCredentialsProvider
# Appended to the user agent of the KCL. Does not impact the functionality of the
# KCL in any other way.

22
docs/multilang/configuring-credential-providers.md
View file

@ -8,30 +8,33 @@ This document should help multilang customers configure a suitable `CredentialPr
## Sample Provider Configuration
In a Properties file, an `AWSCredentialsProperty` configuration might look like:
In a Properties file, an `AwsCredentialsProperty` configuration might look like:
```
AWSCredentialsProvider = StsAssumeRoleCredentialsProvider|<arn>|<sessionName>
AwsCredentialsProvider = StsAssumeRoleCredentialsProvider|<arn>|<sessionName>
```
This basic configuration creates an [StsAssumeRoleCredentialsProvider][sts-assume-provider] with an ARN and session name.
The providers generated by this config property will be [AWS SDK v2 AwsCredentialsProviders][aws-credentials-provider].
These differ from the SDK v1 AWSCredentialsProviders in a number of ways. See [Credentials Provider Changes][credentials-provider-changes].
While functional, this configuration is limited.
For example, this configuration cannot set a regional endpoint (e.g., VPC use case).
Leveraging nested properties, an `AWSCredentialsProperty` value might change to:
Leveraging nested properties, an `AwsCredentialsProperty` value might change to:
```
AWSCredentialsProvider = KclSTSAssumeRoleSessionCredentialsProvider|<arn>|<sessionName>\
AwsCredentialsProvider = KclSTSAssumeRoleSessionCredentialsProvider|<arn>|<sessionName>\
|endpointRegion=us-east-1|externalId=spartacus
```
N.B. Backslash (`\`) is for multi-line legibility and is not required.
You can create a default [DefaultCredentialsProvider][default-credentials-provider] by passing it in the config like:
```
AWSCredentialsProvider = DefaultCredentialsProvider
AwsCredentialsProvider = DefaultCredentialsProvider
```
## Nested Properties
KCL multilang supports "nested properties" on the `AWSCredentialsProvider` key in the properties file.
KCL multilang supports "nested properties" on the `AwsCredentialsProvider` key in the properties file.
The [Backus-Naur form][bnf] of the value:
```
<property-value> ::= <provider-class> ["|" <required-param>]* ["|" <nested-property>]*
@ -61,9 +64,9 @@ A backwards-compatible addition might look like:
}
```
Leveraging nested properties, an `AWSCredentialsProperty` value might look like:
Leveraging nested properties, an `AwsCredentialsProperty` value might look like:
```
AWSCredentialsProvider = KclSTSAssumeRoleSessionCredentialsProvider|<arn>|<sessionName>\
AwsCredentialsProvider = KclSTSAssumeRoleSessionCredentialsProvider|<arn>|<sessionName>\
|endpointRegion=us-east-1|externalId=spartacus
```
@ -73,7 +76,7 @@ N.B. Backslash (`\`) is for multi-line legibility and is not required.
KCL multilang includes a [custom nested property processor for `StsAssumeRole`][kcl-sts-provider].
Multilang configurations that use `StsAssumeRoleSessionCredentialsProvider` need only prefix `Kcl` to exercise this new provider:
```
AWSCredentialsProvider = KclStsAssumeRoleCredentialsProvider|<arn>|<sessionName>
AwsCredentialsProvider = KclStsAssumeRoleCredentialsProvider|<arn>|<sessionName>
```
[aws-credentials-provider]: https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/auth/credentials/AwsCredentialsProvider.html
@ -84,3 +87,4 @@ AWSCredentialsProvider = KclStsAssumeRoleCredentialsProvider|<arn>|<sessionName>
[sts-assume-provider]: https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/sts/auth/StsAssumeRoleCredentialsProvider.html
[profile-credentials-provider-create]: https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/auth/credentials/ProfileCredentialsProvider.html#create(java.lang.String)
[default-credentials-provider]: https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/auth/credentials/DefaultCredentialsProvider.html
[credentials-provider-changes]: https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/migration-client-credentials.html