From a8b70da3b3af5552ece37cec48b231ae2b83e685 Mon Sep 17 00:00:00 2001 From: Ethan Katnic Date: Mon, 19 Aug 2024 17:39:30 -0700 Subject: [PATCH 01/16] Upgrade NestedPropertyKey and NestedPropertyProcessor to AWS SDKv2 --- .../amazon/kinesis/multilang/NestedPropertyKey.java | 4 ++-- .../amazon/kinesis/multilang/NestedPropertyProcessor.java | 6 +++--- .../amazon/kinesis/multilang/NestedPropertyKeyTest.java | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/NestedPropertyKey.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/NestedPropertyKey.java index 19211822..9902d28c 100644 --- a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/NestedPropertyKey.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/NestedPropertyKey.java @@ -17,11 +17,11 @@ package software.amazon.kinesis.multilang; import java.util.HashMap; import java.util.Map; -import com.amazonaws.regions.Regions; import com.google.common.base.CaseFormat; import lombok.AccessLevel; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import software.amazon.awssdk.regions.Region; /** * Key-Value pairs which may be nested in, and extracted from, a property value @@ -74,7 +74,7 @@ public enum NestedPropertyKey { */ ENDPOINT_REGION { void visit(final NestedPropertyProcessor processor, final String region) { - processor.acceptEndpointRegion(Regions.fromName(region)); + processor.acceptEndpointRegion(Region.of(region)); } }, diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/NestedPropertyProcessor.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/NestedPropertyProcessor.java index f7587297..a0b9c3b4 100644 --- a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/NestedPropertyProcessor.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/NestedPropertyProcessor.java @@ -14,7 +14,7 @@ */ package software.amazon.kinesis.multilang; -import com.amazonaws.regions.Regions; +import software.amazon.awssdk.regions.Region; /** * Defines methods to process {@link NestedPropertyKey}s. @@ -28,7 +28,7 @@ public interface NestedPropertyProcessor { * (e.g., https://sns.us-west-1.amazonaws.com, sns.us-west-1.amazonaws.com) * @param signingRegion the region to use for SigV4 signing of requests (e.g. us-west-1) * - * @see #acceptEndpointRegion(Regions) + * @see #acceptEndpointRegion(Region) * @see * AwsClientBuilder.EndpointConfiguration */ @@ -42,7 +42,7 @@ public interface NestedPropertyProcessor { * * @see #acceptEndpoint(String, String) */ - void acceptEndpointRegion(Regions region); + void acceptEndpointRegion(Region region); /** * Set the external id, an optional field to designate who can assume an IAM role. diff --git a/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/NestedPropertyKeyTest.java b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/NestedPropertyKeyTest.java index fbffee81..3c2de9c9 100644 --- a/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/NestedPropertyKeyTest.java +++ b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/NestedPropertyKeyTest.java @@ -14,11 +14,11 @@ */ package software.amazon.kinesis.multilang; -import com.amazonaws.regions.Regions; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; +import software.amazon.awssdk.regions.Region; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.verify; @@ -64,9 +64,9 @@ public class NestedPropertyKeyTest { @Test public void testEndpointRegion() { - final Regions expectedRegion = Regions.GovCloud; + final Region expectedRegion = Region.US_GOV_WEST_1; - parse(mockProcessor, createKey(ENDPOINT_REGION, expectedRegion.getName())); + parse(mockProcessor, createKey(ENDPOINT_REGION, expectedRegion.id())); verify(mockProcessor).acceptEndpointRegion(expectedRegion); } From 119ef4220607a3d15af5d82d41178390e320bea0 Mon Sep 17 00:00:00 2001 From: Ethan Katnic Date: Mon, 19 Aug 2024 17:41:06 -0700 Subject: [PATCH 02/16] Rewrite KCLSTSAssumeRoleCredentialsProvider to use AWS SDK v2 --- ...SAssumeRoleSessionCredentialsProvider.java | 86 ------------------- .../KclStsAssumeRoleCredentialsProvider.java | 53 ++++++++++++ ...tsAssumeRoleCredentialsProviderConfig.java | 24 ++++++ ...sAssumeRoleCredentialsProviderFactory.java | 40 +++++++++ 4 files changed, 117 insertions(+), 86 deletions(-) delete mode 100644 amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/KclSTSAssumeRoleSessionCredentialsProvider.java create mode 100644 amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/KclStsAssumeRoleCredentialsProvider.java create mode 100644 amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/StsAssumeRoleCredentialsProviderConfig.java create mode 100644 amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/StsAssumeRoleCredentialsProviderFactory.java diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/KclSTSAssumeRoleSessionCredentialsProvider.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/KclSTSAssumeRoleSessionCredentialsProvider.java deleted file mode 100644 index b5b9f924..00000000 --- a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/KclSTSAssumeRoleSessionCredentialsProvider.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright 2023 Amazon.com, Inc. or its affiliates. - * Licensed under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package software.amazon.kinesis.multilang.auth; - -import java.util.Arrays; - -import com.amazonaws.auth.AWSSessionCredentials; -import com.amazonaws.auth.AWSSessionCredentialsProvider; -import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider; -import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider.Builder; -import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration; -import com.amazonaws.regions.Regions; -import com.amazonaws.services.securitytoken.AWSSecurityTokenService; -import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient; -import software.amazon.kinesis.multilang.NestedPropertyKey; -import software.amazon.kinesis.multilang.NestedPropertyProcessor; - -/** - * An {@link AWSSessionCredentialsProvider} that is backed by STSAssumeRole. - */ -public class KclSTSAssumeRoleSessionCredentialsProvider - implements AWSSessionCredentialsProvider, NestedPropertyProcessor { - - private final Builder builder; - - private final STSAssumeRoleSessionCredentialsProvider provider; - - /** - * - * @param params vararg parameters which must include roleArn at index=0, - * and roleSessionName at index=1 - */ - public KclSTSAssumeRoleSessionCredentialsProvider(final String[] params) { - this(params[0], params[1], Arrays.copyOfRange(params, 2, params.length)); - } - - public KclSTSAssumeRoleSessionCredentialsProvider( - final String roleArn, final String roleSessionName, final String... params) { - builder = new Builder(roleArn, roleSessionName); - NestedPropertyKey.parse(this, params); - provider = builder.build(); - } - - @Override - public AWSSessionCredentials getCredentials() { - return provider.getCredentials(); - } - - @Override - public void refresh() { - // do nothing - } - - @Override - public void acceptEndpoint(final String serviceEndpoint, final String signingRegion) { - final EndpointConfiguration endpoint = new EndpointConfiguration(serviceEndpoint, signingRegion); - final AWSSecurityTokenService stsClient = AWSSecurityTokenServiceClient.builder() - .withEndpointConfiguration(endpoint) - .build(); - builder.withStsClient(stsClient); - } - - @Override - public void acceptEndpointRegion(final Regions region) { - final AWSSecurityTokenService stsClient = - AWSSecurityTokenServiceClient.builder().withRegion(region).build(); - builder.withStsClient(stsClient); - } - - @Override - public void acceptExternalId(final String externalId) { - builder.withExternalId(externalId); - } -} diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/KclStsAssumeRoleCredentialsProvider.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/KclStsAssumeRoleCredentialsProvider.java new file mode 100644 index 00000000..d077e6f2 --- /dev/null +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/KclStsAssumeRoleCredentialsProvider.java @@ -0,0 +1,53 @@ +package software.amazon.kinesis.multilang.auth; + +import java.util.Arrays; + +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider; +import software.amazon.kinesis.multilang.NestedPropertyKey; +import software.amazon.kinesis.multilang.NestedPropertyProcessor; + +public class KclStsAssumeRoleCredentialsProvider implements AwsCredentialsProvider, NestedPropertyProcessor { + private final String roleArn; + private final String roleSessionName; + private Region region; + private String serviceEndpoint; + private String externalId; + + public KclStsAssumeRoleCredentialsProvider(String[] params) { + this(params[0], params[1], Arrays.copyOfRange(params, 2, params.length)); + } + + public KclStsAssumeRoleCredentialsProvider(String roleArn, String roleSessionName, String... params) { + this.roleArn = roleArn; + this.roleSessionName = roleSessionName; + NestedPropertyKey.parse(this, params); + } + + @Override + public AwsCredentials resolveCredentials() { + StsAssumeRoleCredentialsProviderConfig config = new StsAssumeRoleCredentialsProviderConfig( + roleArn, roleSessionName, region, serviceEndpoint, externalId); + StsAssumeRoleCredentialsProvider stsAssumeRoleCredentialsProvider = + StsAssumeRoleCredentialsProviderFactory.createProvider(config); + return stsAssumeRoleCredentialsProvider.resolveCredentials(); + } + + @Override + public void acceptEndpoint(String serviceEndpoint, String signingRegion) { + this.serviceEndpoint = serviceEndpoint; + this.region = Region.of(signingRegion); + } + + @Override + public void acceptEndpointRegion(Region region) { + this.region = region; + } + + @Override + public void acceptExternalId(String externalId) { + this.externalId = externalId; + } +} diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/StsAssumeRoleCredentialsProviderConfig.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/StsAssumeRoleCredentialsProviderConfig.java new file mode 100644 index 00000000..f4e41257 --- /dev/null +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/StsAssumeRoleCredentialsProviderConfig.java @@ -0,0 +1,24 @@ +package software.amazon.kinesis.multilang.auth; + +import lombok.Getter; +import lombok.Setter; +import software.amazon.awssdk.regions.Region; + +@Getter +@Setter +public class StsAssumeRoleCredentialsProviderConfig { + private final String roleArn; + private final String roleSessionName; + private final Region region; + private final String serviceEndpoint; + private final String externalId; + + public StsAssumeRoleCredentialsProviderConfig( + String roleArn, String roleSessionName, Region region, String serviceEndpoint, String externalId) { + this.roleArn = roleArn; + this.roleSessionName = roleSessionName; + this.region = region; + this.serviceEndpoint = serviceEndpoint; + this.externalId = externalId; + } +} diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/StsAssumeRoleCredentialsProviderFactory.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/StsAssumeRoleCredentialsProviderFactory.java new file mode 100644 index 00000000..b3b55531 --- /dev/null +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/StsAssumeRoleCredentialsProviderFactory.java @@ -0,0 +1,40 @@ +package software.amazon.kinesis.multilang.auth; + +import java.net.URI; +import java.net.URISyntaxException; + +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.StsClientBuilder; +import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider; +import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; + +public class StsAssumeRoleCredentialsProviderFactory { + + public static StsAssumeRoleCredentialsProvider createProvider(StsAssumeRoleCredentialsProviderConfig config) { + StsClientBuilder stsClientBuilder = StsClient.builder(); + + if (config.getRegion() != null) { + stsClientBuilder.region(config.getRegion()); + } + + if (config.getServiceEndpoint() != null) { + try { + stsClientBuilder.endpointOverride(new URI(config.getServiceEndpoint())); + } catch (URISyntaxException e) { + throw new IllegalArgumentException("Invalid service endpoint: " + config.getServiceEndpoint(), e); + } + } + + StsClient stsClient = stsClientBuilder.build(); + + AssumeRoleRequest assumeRoleRequest = AssumeRoleRequest.builder() + .roleArn(config.getRoleArn()) + .roleSessionName(config.getRoleSessionName()) + .build(); + + return StsAssumeRoleCredentialsProvider.builder() + .refreshRequest(assumeRoleRequest) + .stsClient(stsClient) + .build(); + } +} From 17f7d5184565cdb9a976736d53a70c2839614c0b Mon Sep 17 00:00:00 2001 From: Ethan Katnic Date: Tue, 20 Aug 2024 10:08:10 -0700 Subject: [PATCH 03/16] Remove v1 credential wrapper from daemon configuration --- amazon-kinesis-client-multilang/pom.xml | 26 ------- ...edentialsProviderPropertyValueDecoder.java | 38 +++++----- .../config/MultiLangDaemonConfiguration.java | 5 +- .../credentials/V2CredentialWrapper.java | 52 -------------- .../multilang/NestedPropertyKeyTest.java | 5 +- ...umeRoleSessionCredentialsProviderTest.java | 7 +- ...tialsProviderPropertyValueDecoderTest.java | 72 ++++++++----------- .../KinesisClientLibConfiguratorTest.java | 52 +++++--------- 8 files changed, 75 insertions(+), 182 deletions(-) delete mode 100644 amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/credentials/V2CredentialWrapper.java diff --git a/amazon-kinesis-client-multilang/pom.xml b/amazon-kinesis-client-multilang/pom.xml index c972b61c..17663f32 100644 --- a/amazon-kinesis-client-multilang/pom.xml +++ b/amazon-kinesis-client-multilang/pom.xml @@ -43,32 +43,6 @@ ${awssdk.version} - - com.amazonaws - aws-java-sdk-core - ${aws-java-sdk.version} - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.dataformat - jackson-dataformat-cbor - - - org.apache.httpcomponents - httpclient - - - - - - com.amazonaws - aws-java-sdk-sts - ${aws-java-sdk.version} - - org.projectlombok lombok diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoder.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoder.java index 8110d4f7..9f9d5315 100644 --- a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoder.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoder.java @@ -22,15 +22,15 @@ import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.AWSCredentialsProviderChain; import lombok.extern.slf4j.Slf4j; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain; /** * Get AWSCredentialsProvider property. */ @Slf4j -class AWSCredentialsProviderPropertyValueDecoder implements IPropertyValueDecoder { +class AWSCredentialsProviderPropertyValueDecoder implements IPropertyValueDecoder { private static final String LIST_DELIMITER = ","; private static final String ARG_DELIMITER = "|"; @@ -47,13 +47,15 @@ class AWSCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode * @return corresponding variable in correct type */ @Override - public AWSCredentialsProvider decodeValue(String value) { + public AwsCredentialsProvider decodeValue(String value) { if (value != null) { List providerNames = getProviderNames(value); - List providers = getValidCredentialsProviders(providerNames); - AWSCredentialsProvider[] ps = new AWSCredentialsProvider[providers.size()]; + List providers = getValidCredentialsProviders(providerNames); + AwsCredentialsProvider[] ps = new AwsCredentialsProvider[providers.size()]; providers.toArray(ps); - return new AWSCredentialsProviderChain(providers); + return AwsCredentialsProviderChain.builder() + .credentialsProviders(providers) + .build(); } else { throw new IllegalArgumentException("Property AWSCredentialsProvider is missing."); } @@ -63,25 +65,25 @@ class AWSCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode * @return list of supported types */ @Override - public List> getSupportedTypes() { - return Collections.singletonList(AWSCredentialsProvider.class); + public List> getSupportedTypes() { + return Collections.singletonList(AwsCredentialsProvider.class); } /** * Convert string list to a list of valid credentials providers. */ - private static List getValidCredentialsProviders(List providerNames) { - List credentialsProviders = new ArrayList<>(); + private static List getValidCredentialsProviders(List providerNames) { + List credentialsProviders = new ArrayList<>(); for (String providerName : providerNames) { final String[] nameAndArgs = providerName.split("\\" + ARG_DELIMITER); - final Class clazz; + final Class clazz; try { final Class c = Class.forName(nameAndArgs[0]); - if (!AWSCredentialsProvider.class.isAssignableFrom(c)) { + if (!AwsCredentialsProvider.class.isAssignableFrom(c)) { continue; } - clazz = (Class) c; + clazz = (Class) c; } catch (ClassNotFoundException cnfe) { // Providers are a product of prefixed Strings to cover multiple // namespaces (e.g., "Foo" -> { "some.auth.Foo", "kcl.auth.Foo" }). @@ -90,7 +92,7 @@ class AWSCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode } log.info("Attempting to construct {}", clazz); - AWSCredentialsProvider provider = null; + AwsCredentialsProvider provider = null; if (nameAndArgs.length > 1) { final String[] varargs = Arrays.copyOfRange(nameAndArgs, 1, nameAndArgs.length); @@ -153,13 +155,13 @@ class AWSCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode } @FunctionalInterface - private interface CredentialsProviderConstructor { + private interface CredentialsProviderConstructor { T construct() throws IllegalAccessException, InstantiationException, InvocationTargetException, NoSuchMethodException; } /** - * Attempts to construct an {@link AWSCredentialsProvider}. + * Attempts to construct an {@link AwsCredentialsProvider}. * * @param providerName Raw, unmodified provider name. Should there be an * Exeception during construction, this parameter will be logged. @@ -168,7 +170,7 @@ class AWSCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode * * @param type of the CredentialsProvider to construct */ - private static T constructProvider( + private static T constructProvider( final String providerName, final CredentialsProviderConstructor constructor) { try { return constructor.construct(); diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/MultiLangDaemonConfiguration.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/MultiLangDaemonConfiguration.java index 3336be88..2225da7d 100644 --- a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/MultiLangDaemonConfiguration.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/MultiLangDaemonConfiguration.java @@ -55,7 +55,6 @@ import software.amazon.kinesis.leases.ShardPrioritization; import software.amazon.kinesis.lifecycle.LifecycleConfig; import software.amazon.kinesis.metrics.MetricsConfig; import software.amazon.kinesis.metrics.MetricsLevel; -import software.amazon.kinesis.multilang.config.credentials.V2CredentialWrapper; import software.amazon.kinesis.processor.ProcessorConfig; import software.amazon.kinesis.processor.ShardRecordProcessorFactory; import software.amazon.kinesis.retrieval.RetrievalConfig; @@ -282,9 +281,7 @@ public class MultiLangDaemonConfiguration { ArrayConverter arrayConverter = new ArrayConverter(String[].class, new StringConverter()); arrayConverter.setDelimiter(','); convertUtilsBean.register(arrayConverter, String[].class); - AWSCredentialsProviderPropertyValueDecoder oldCredentialsDecoder = - new AWSCredentialsProviderPropertyValueDecoder(); - Function converter = s -> new V2CredentialWrapper(oldCredentialsDecoder.decodeValue(s)); + Function converter = s -> s; this.kinesisCredentialsProvider = new BuilderDynaBean( AwsCredentialsProvider.class, convertUtilsBean, converter, CREDENTIALS_DEFAULT_SEARCH_PATH); diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/credentials/V2CredentialWrapper.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/credentials/V2CredentialWrapper.java deleted file mode 100644 index e1b6072a..00000000 --- a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/credentials/V2CredentialWrapper.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. - * Licensed under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package software.amazon.kinesis.multilang.config.credentials; - -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.AWSSessionCredentials; -import lombok.RequiredArgsConstructor; -import software.amazon.awssdk.auth.credentials.AwsCredentials; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; - -@RequiredArgsConstructor -public class V2CredentialWrapper implements AwsCredentialsProvider { - - private final AWSCredentialsProvider oldCredentialsProvider; - - @Override - public AwsCredentials resolveCredentials() { - AWSCredentials current = oldCredentialsProvider.getCredentials(); - if (current instanceof AWSSessionCredentials) { - return AwsSessionCredentials.create( - current.getAWSAccessKeyId(), - current.getAWSSecretKey(), - ((AWSSessionCredentials) current).getSessionToken()); - } - return new AwsCredentials() { - @Override - public String accessKeyId() { - return current.getAWSAccessKeyId(); - } - - @Override - public String secretAccessKey() { - return current.getAWSSecretKey(); - } - }; - } -} diff --git a/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/NestedPropertyKeyTest.java b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/NestedPropertyKeyTest.java index 3c2de9c9..350b9fac 100644 --- a/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/NestedPropertyKeyTest.java +++ b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/NestedPropertyKeyTest.java @@ -70,7 +70,10 @@ public class NestedPropertyKeyTest { verify(mockProcessor).acceptEndpointRegion(expectedRegion); } - @Test(expected = IllegalArgumentException.class) + // Region.of(), which is invoked in this test, no longer throws an IllegalArgumentException when given an invalid + // region + // We would need to implement our own region validation to maintain this test + // @Test(expected = IllegalArgumentException.class) public void testInvalidEndpointRegion() { parse(mockProcessor, createKey(ENDPOINT_REGION, "snuffleupagus")); } diff --git a/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/auth/KclSTSAssumeRoleSessionCredentialsProviderTest.java b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/auth/KclSTSAssumeRoleSessionCredentialsProviderTest.java index c27a425d..c479f77a 100644 --- a/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/auth/KclSTSAssumeRoleSessionCredentialsProviderTest.java +++ b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/auth/KclSTSAssumeRoleSessionCredentialsProviderTest.java @@ -31,15 +31,14 @@ public class KclSTSAssumeRoleSessionCredentialsProviderTest { */ @Test public void testConstructorWithoutOptionalParams() { - new KclSTSAssumeRoleSessionCredentialsProvider(new String[] {ARN, SESSION_NAME}); + new KclStsAssumeRoleCredentialsProvider(new String[] {ARN, SESSION_NAME}); } @Test public void testAcceptEndpoint() { // discovered exception during e2e testing; therefore, this test is // to simply verify the constructed STS client doesn't go *boom* - final KclSTSAssumeRoleSessionCredentialsProvider provider = - new KclSTSAssumeRoleSessionCredentialsProvider(ARN, SESSION_NAME); + final KclStsAssumeRoleCredentialsProvider provider = new KclStsAssumeRoleCredentialsProvider(ARN, SESSION_NAME); provider.acceptEndpoint("endpoint", "us-east-1"); } @@ -53,7 +52,7 @@ public class KclSTSAssumeRoleSessionCredentialsProviderTest { } } - private static class VarArgsSpy extends KclSTSAssumeRoleSessionCredentialsProvider { + private static class VarArgsSpy extends KclStsAssumeRoleCredentialsProvider { private String externalId; diff --git a/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoderTest.java b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoderTest.java index ba5a0925..8039194d 100644 --- a/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoderTest.java +++ b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoderTest.java @@ -16,16 +16,16 @@ package software.amazon.kinesis.multilang.config; import java.util.Arrays; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.AWSCredentialsProviderChain; -import com.amazonaws.auth.BasicAWSCredentials; import lombok.ToString; import org.hamcrest.Description; import org.hamcrest.Matcher; import org.hamcrest.TypeSafeDiagnosingMatcher; import org.junit.Test; -import software.amazon.kinesis.multilang.auth.KclSTSAssumeRoleSessionCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain; +import software.amazon.kinesis.multilang.auth.KclStsAssumeRoleCredentialsProvider; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; @@ -43,7 +43,7 @@ public class AWSCredentialsProviderPropertyValueDecoderTest { private final AWSCredentialsProviderPropertyValueDecoder decoder = new AWSCredentialsProviderPropertyValueDecoder(); @ToString - private static class AWSCredentialsMatcher extends TypeSafeDiagnosingMatcher { + private static class AWSCredentialsMatcher extends TypeSafeDiagnosingMatcher { private final Matcher akidMatcher; private final Matcher secretMatcher; @@ -52,12 +52,12 @@ public class AWSCredentialsProviderPropertyValueDecoderTest { public AWSCredentialsMatcher(String akid, String secret) { this.akidMatcher = equalTo(akid); this.secretMatcher = equalTo(secret); - this.classMatcher = instanceOf(AWSCredentialsProviderChain.class); + this.classMatcher = instanceOf(AwsCredentialsProviderChain.class); } @Override - protected boolean matchesSafely(AWSCredentialsProvider item, Description mismatchDescription) { - AWSCredentials actual = item.getCredentials(); + protected boolean matchesSafely(AwsCredentialsProvider item, Description mismatchDescription) { + AwsCredentials actual = item.resolveCredentials(); boolean matched = true; if (!classMatcher.matches(item)) { @@ -65,12 +65,12 @@ public class AWSCredentialsProviderPropertyValueDecoderTest { matched = false; } - if (!akidMatcher.matches(actual.getAWSAccessKeyId())) { - akidMatcher.describeMismatch(actual.getAWSAccessKeyId(), mismatchDescription); + if (!akidMatcher.matches(actual.accessKeyId())) { + akidMatcher.describeMismatch(actual.accessKeyId(), mismatchDescription); matched = false; } - if (!secretMatcher.matches(actual.getAWSSecretKey())) { - secretMatcher.describeMismatch(actual.getAWSSecretKey(), mismatchDescription); + if (!secretMatcher.matches(actual.secretAccessKey())) { + secretMatcher.describeMismatch(actual.secretAccessKey(), mismatchDescription); matched = false; } return matched; @@ -90,25 +90,25 @@ public class AWSCredentialsProviderPropertyValueDecoderTest { @Test public void testSingleProvider() { - AWSCredentialsProvider provider = decoder.decodeValue(credentialName1); + AwsCredentialsProvider provider = decoder.decodeValue(credentialName1); assertThat(provider, hasCredentials(TEST_ACCESS_KEY_ID, TEST_SECRET_KEY)); } @Test public void testTwoProviders() { - AWSCredentialsProvider provider = decoder.decodeValue(credentialName1 + "," + credentialName1); + AwsCredentialsProvider provider = decoder.decodeValue(credentialName1 + "," + credentialName1); assertThat(provider, hasCredentials(TEST_ACCESS_KEY_ID, TEST_SECRET_KEY)); } @Test public void testProfileProviderWithOneArg() { - AWSCredentialsProvider provider = decoder.decodeValue(credentialName2 + "|arg"); + AwsCredentialsProvider provider = decoder.decodeValue(credentialName2 + "|arg"); assertThat(provider, hasCredentials("arg", "blank")); } @Test public void testProfileProviderWithTwoArgs() { - AWSCredentialsProvider provider = decoder.decodeValue(credentialName2 + "|arg1|arg2"); + AwsCredentialsProvider provider = decoder.decodeValue(credentialName2 + "|arg1|arg2"); assertThat(provider, hasCredentials("arg1", "arg2")); } @@ -118,10 +118,10 @@ public class AWSCredentialsProviderPropertyValueDecoderTest { @Test public void testKclAuthProvider() { for (final String className : Arrays.asList( - KclSTSAssumeRoleSessionCredentialsProvider.class.getName(), // fully-qualified name - KclSTSAssumeRoleSessionCredentialsProvider.class.getSimpleName() // name-only; needs prefix + KclStsAssumeRoleCredentialsProvider.class.getName(), // fully-qualified name + KclStsAssumeRoleCredentialsProvider.class.getSimpleName() // name-only; needs prefix )) { - final AWSCredentialsProvider provider = decoder.decodeValue(className + "|arn|sessionName"); + final AwsCredentialsProvider provider = decoder.decodeValue(className + "|arn|sessionName"); assertNotNull(className, provider); } } @@ -135,28 +135,24 @@ public class AWSCredentialsProviderPropertyValueDecoderTest { final String className = VarArgCredentialsProvider.class.getName(); final String encodedValue = className + "|" + String.join("|", args); - final AWSCredentialsProvider provider = decoder.decodeValue(encodedValue); - assertEquals(Arrays.toString(args), provider.getCredentials().getAWSAccessKeyId()); + final AwsCredentialsProvider provider = decoder.decodeValue(encodedValue); + assertEquals(Arrays.toString(args), provider.resolveCredentials().accessKeyId()); } /** * This credentials provider will always succeed */ - public static class AlwaysSucceedCredentialsProvider implements AWSCredentialsProvider { - + public static class AlwaysSucceedCredentialsProvider implements AwsCredentialsProvider { @Override - public AWSCredentials getCredentials() { - return new BasicAWSCredentials(TEST_ACCESS_KEY_ID, TEST_SECRET_KEY); + public AwsCredentials resolveCredentials() { + return AwsBasicCredentials.create(TEST_ACCESS_KEY_ID, TEST_SECRET_KEY); } - - @Override - public void refresh() {} } /** * This credentials provider needs a constructor call to instantiate it */ - public static class ConstructorCredentialsProvider implements AWSCredentialsProvider { + public static class ConstructorCredentialsProvider implements AwsCredentialsProvider { private String arg1; private String arg2; @@ -172,15 +168,12 @@ public class AWSCredentialsProviderPropertyValueDecoderTest { } @Override - public AWSCredentials getCredentials() { - return new BasicAWSCredentials(arg1, arg2); + public AwsCredentials resolveCredentials() { + return AwsBasicCredentials.create(arg1, arg2); } - - @Override - public void refresh() {} } - private static class VarArgCredentialsProvider implements AWSCredentialsProvider { + private static class VarArgCredentialsProvider implements AwsCredentialsProvider { private final String[] args; @@ -189,13 +182,10 @@ public class AWSCredentialsProviderPropertyValueDecoderTest { } @Override - public AWSCredentials getCredentials() { + public AwsCredentials resolveCredentials() { // KISS solution to surface the constructor args final String flattenedArgs = Arrays.toString(args); - return new BasicAWSCredentials(flattenedArgs, flattenedArgs); + return AwsBasicCredentials.create(flattenedArgs, flattenedArgs); } - - @Override - public void refresh() {} } } diff --git a/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/KinesisClientLibConfiguratorTest.java b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/KinesisClientLibConfiguratorTest.java index b0e3b870..4ac6521e 100644 --- a/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/KinesisClientLibConfiguratorTest.java +++ b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/KinesisClientLibConfiguratorTest.java @@ -22,15 +22,14 @@ import java.util.Date; import java.util.HashSet; import java.util.Set; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; import com.google.common.collect.ImmutableSet; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.runners.MockitoJUnitRunner; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.kinesis.common.InitialPositionInStream; import software.amazon.kinesis.metrics.MetricsLevel; @@ -526,71 +525,52 @@ public class KinesisClientLibConfiguratorTest { /** * This credentials provider will always succeed */ - public static class AlwaysSucceedCredentialsProvider implements AWSCredentialsProvider { - + public static class AlwaysSucceedCredentialsProvider implements AwsCredentialsProvider { @Override - public AWSCredentials getCredentials() { - return new BasicAWSCredentials("a", "b"); + public AwsCredentials resolveCredentials() { + return AwsBasicCredentials.create("a", "b"); } - - @Override - public void refresh() {} } /** * This credentials provider will always succeed */ - public static class AlwaysSucceedCredentialsProviderKinesis implements AWSCredentialsProvider { - + public static class AlwaysSucceedCredentialsProviderKinesis implements AwsCredentialsProvider { @Override - public AWSCredentials getCredentials() { - return new BasicAWSCredentials("", ""); + public AwsCredentials resolveCredentials() { + return AwsBasicCredentials.create("", ""); } - - @Override - public void refresh() {} } /** * This credentials provider will always succeed */ - public static class AlwaysSucceedCredentialsProviderDynamoDB implements AWSCredentialsProvider { - + public static class AlwaysSucceedCredentialsProviderDynamoDB implements AwsCredentialsProvider { @Override - public AWSCredentials getCredentials() { - return new BasicAWSCredentials("", ""); + public AwsCredentials resolveCredentials() { + return AwsBasicCredentials.create("", ""); } - - @Override - public void refresh() {} } /** * This credentials provider will always succeed */ - public static class AlwaysSucceedCredentialsProviderCloudWatch implements AWSCredentialsProvider { - + public static class AlwaysSucceedCredentialsProviderCloudWatch implements AwsCredentialsProvider { @Override - public AWSCredentials getCredentials() { - return new BasicAWSCredentials("", ""); + public AwsCredentials resolveCredentials() { + return AwsBasicCredentials.create("", ""); } - - @Override - public void refresh() {} } /** * This credentials provider will always fail */ - public static class AlwaysFailCredentialsProvider implements AWSCredentialsProvider { + public static class AlwaysFailCredentialsProvider implements AwsCredentialsProvider { @Override - public AWSCredentials getCredentials() { + public AwsCredentials resolveCredentials() { throw new IllegalArgumentException(); } - - @Override - public void refresh() {} } private MultiLangDaemonConfiguration getConfiguration(String configString) { From c8f642b435298b5e5bf2b24da31c584148f2f0c1 Mon Sep 17 00:00:00 2001 From: Ethan Katnic Date: Wed, 21 Aug 2024 08:48:31 -0700 Subject: [PATCH 04/16] Upgrade ProviderPropertyValueDecoder to aws SDK v2 --- ...dentialsProviderPropertyValueDecoder.java} | 49 ++++++++++++++----- .../config/MultiLangDaemonConfiguration.java | 5 +- ...tialsProviderPropertyValueDecoderTest.java | 2 +- 3 files changed, 40 insertions(+), 16 deletions(-) rename amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/{AWSCredentialsProviderPropertyValueDecoder.java => AwsCredentialsProviderPropertyValueDecoder.java} (78%) diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoder.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AwsCredentialsProviderPropertyValueDecoder.java similarity index 78% rename from amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoder.java rename to amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AwsCredentialsProviderPropertyValueDecoder.java index 9f9d5315..536c0b1b 100644 --- a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoder.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AwsCredentialsProviderPropertyValueDecoder.java @@ -15,6 +15,8 @@ package software.amazon.kinesis.multilang.config; import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -30,14 +32,14 @@ import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain; * Get AWSCredentialsProvider property. */ @Slf4j -class AWSCredentialsProviderPropertyValueDecoder implements IPropertyValueDecoder { +class AwsCredentialsProviderPropertyValueDecoder implements IPropertyValueDecoder { private static final String LIST_DELIMITER = ","; private static final String ARG_DELIMITER = "|"; /** * Constructor. */ - AWSCredentialsProviderPropertyValueDecoder() {} + AwsCredentialsProviderPropertyValueDecoder() {} /** * Get AWSCredentialsProvider property. @@ -97,22 +99,27 @@ class AWSCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode final String[] varargs = Arrays.copyOfRange(nameAndArgs, 1, nameAndArgs.length); // attempt to invoke an explicit N-arg constructor of FooClass(String, String, ...) - provider = constructProvider(providerName, () -> { - Class[] argTypes = new Class[nameAndArgs.length - 1]; - Arrays.fill(argTypes, String.class); - return clazz.getConstructor(argTypes).newInstance(varargs); - }); + provider = constructProvider( + providerName, + () -> { + Class[] argTypes = new Class[nameAndArgs.length - 1]; + Arrays.fill(argTypes, String.class); + return clazz.getConstructor(argTypes).newInstance(varargs); + }, + clazz); if (provider == null) { // attempt to invoke a public varargs/array constructor of FooClass(String[]) - provider = constructProvider(providerName, () -> clazz.getConstructor(String[].class) - .newInstance((Object) varargs)); + provider = constructProvider( + providerName, + () -> clazz.getConstructor(String[].class).newInstance((Object) varargs), + clazz); } } if (provider == null) { // regardless of parameters, fallback to invoke a public no-arg constructor - provider = constructProvider(providerName, clazz::newInstance); + provider = constructProvider(providerName, clazz::newInstance, clazz); } if (provider != null) { @@ -146,7 +153,7 @@ class AWSCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode // Customer provides a short name of a provider offered by this multi-lang package "software.amazon.kinesis.multilang.auth.", - + "software.amazon.awssdk.auth.credentials.", // Customer provides a fully-qualified provider name, or a custom credentials provider // (e.g., com.amazonaws.auth.ClasspathFileCredentialsProvider, org.mycompany.FooProvider) "") @@ -171,12 +178,28 @@ class AWSCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode * @param type of the CredentialsProvider to construct */ private static T constructProvider( - final String providerName, final CredentialsProviderConstructor constructor) { + final String providerName, + final CredentialsProviderConstructor constructor, + Class clazz) { try { + // Try to create an instance using the given constructor return constructor.construct(); + } catch (InstantiationException e) { + try { + Method createMethod = clazz.getDeclaredMethod("create"); + if (Modifier.isStatic(createMethod.getModifiers())) { + return (T) createMethod.invoke(null); + } else { + log.warn("Found non-static create() method in {}", providerName); + } + } catch (NoSuchMethodException ignored) { + // Ignore if create() method is not found + } catch (IllegalAccessException | InvocationTargetException e1) { + log.warn("Exception thrown by create() method in {}", providerName, e1.getCause()); + } } catch (NoSuchMethodException ignored) { // ignore - } catch (IllegalAccessException | InstantiationException | InvocationTargetException | RuntimeException e) { + } catch (IllegalAccessException | InvocationTargetException | RuntimeException e) { log.warn("Failed to construct {}", providerName, e); } return null; diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/MultiLangDaemonConfiguration.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/MultiLangDaemonConfiguration.java index 2225da7d..54d9ac5c 100644 --- a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/MultiLangDaemonConfiguration.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/MultiLangDaemonConfiguration.java @@ -280,8 +280,9 @@ public class MultiLangDaemonConfiguration { ArrayConverter arrayConverter = new ArrayConverter(String[].class, new StringConverter()); arrayConverter.setDelimiter(','); - convertUtilsBean.register(arrayConverter, String[].class); - Function converter = s -> s; + convertUtilsBean.register(arrayConverter, String[].class);AwsCredentialsProviderPropertyValueDecoder credentialsDecoder = + new AwsCredentialsProviderPropertyValueDecoder(); + Function converter = credentialsDecoder::decodeValue; this.kinesisCredentialsProvider = new BuilderDynaBean( AwsCredentialsProvider.class, convertUtilsBean, converter, CREDENTIALS_DEFAULT_SEARCH_PATH); diff --git a/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoderTest.java b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoderTest.java index 8039194d..58bca383 100644 --- a/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoderTest.java +++ b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoderTest.java @@ -40,7 +40,7 @@ public class AWSCredentialsProviderPropertyValueDecoderTest { private final String credentialName1 = AlwaysSucceedCredentialsProvider.class.getName(); private final String credentialName2 = ConstructorCredentialsProvider.class.getName(); - private final AWSCredentialsProviderPropertyValueDecoder decoder = new AWSCredentialsProviderPropertyValueDecoder(); + private final AwsCredentialsProviderPropertyValueDecoder decoder = new AwsCredentialsProviderPropertyValueDecoder(); @ToString private static class AWSCredentialsMatcher extends TypeSafeDiagnosingMatcher { From f83f8c8e02757df710c8fed00303cd66f3fb6906 Mon Sep 17 00:00:00 2001 From: Ethan Katnic Date: Wed, 21 Aug 2024 08:49:42 -0700 Subject: [PATCH 05/16] Upgrade DefaultCredentialsProvider and AwsBasicCredentials to AWS SDK v2 --- .../amazon/kinesis/multilang/MultiLangDaemonConfigTest.java | 4 ++-- .../multilang/config/KinesisClientLibConfiguratorTest.java | 6 +++--- .../src/test/resources/multilang.properties | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MultiLangDaemonConfigTest.java b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MultiLangDaemonConfigTest.java index de5a1405..8a244ee0 100644 --- a/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MultiLangDaemonConfigTest.java +++ b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MultiLangDaemonConfigTest.java @@ -65,7 +65,7 @@ public class MultiLangDaemonConfigTest { String properties = String.format( "executableName = %s\n" + "applicationName = %s\n" - + "AWSCredentialsProvider = DefaultAWSCredentialsProviderChain\n" + + "AWSCredentialsProvider = DefaultCredentialsProvider\n" + "processingLanguage = malbolge\n" + "regionName = %s\n", EXE, APPLICATION_NAME, "us-east-1"); @@ -182,7 +182,7 @@ public class MultiLangDaemonConfigTest { @Test public void testPropertyValidation() { String propertiesNoExecutableName = "applicationName = testApp \n" + "streamName = fakeStream \n" - + "AWSCredentialsProvider = DefaultAWSCredentialsProviderChain\n" + "processingLanguage = malbolge"; + + "AWSCredentialsProvider = DefaultCredentialsProvider\n" + "processingLanguage = malbolge"; ClassLoader classLoader = Mockito.mock(ClassLoader.class); Mockito.doReturn(new ByteArrayInputStream(propertiesNoExecutableName.getBytes())) diff --git a/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/KinesisClientLibConfiguratorTest.java b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/KinesisClientLibConfiguratorTest.java index 4ac6521e..b5c51841 100644 --- a/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/KinesisClientLibConfiguratorTest.java +++ b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/KinesisClientLibConfiguratorTest.java @@ -538,7 +538,7 @@ public class KinesisClientLibConfiguratorTest { public static class AlwaysSucceedCredentialsProviderKinesis implements AwsCredentialsProvider { @Override public AwsCredentials resolveCredentials() { - return AwsBasicCredentials.create("", ""); + return AwsBasicCredentials.create("DUMMY_ACCESS_KEY_ID", "DUMMY_SECRET_ACCESS_KEY"); } } @@ -548,7 +548,7 @@ public class KinesisClientLibConfiguratorTest { public static class AlwaysSucceedCredentialsProviderDynamoDB implements AwsCredentialsProvider { @Override public AwsCredentials resolveCredentials() { - return AwsBasicCredentials.create("", ""); + return AwsBasicCredentials.create("DUMMY_ACCESS_KEY_ID", "DUMMY_SECRET_ACCESS_KEY"); } } @@ -558,7 +558,7 @@ public class KinesisClientLibConfiguratorTest { public static class AlwaysSucceedCredentialsProviderCloudWatch implements AwsCredentialsProvider { @Override public AwsCredentials resolveCredentials() { - return AwsBasicCredentials.create("", ""); + return AwsBasicCredentials.create("DUMMY_ACCESS_KEY_ID", "DUMMY_SECRET_ACCESS_KEY"); } } diff --git a/amazon-kinesis-client-multilang/src/test/resources/multilang.properties b/amazon-kinesis-client-multilang/src/test/resources/multilang.properties index 34cb0c1a..347d4e8a 100644 --- a/amazon-kinesis-client-multilang/src/test/resources/multilang.properties +++ b/amazon-kinesis-client-multilang/src/test/resources/multilang.properties @@ -20,7 +20,7 @@ applicationName = MultiLangTest # The DefaultAWSCredentialsProviderChain checks several other providers, which is # described here: # http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html -AWSCredentialsProvider = DefaultAWSCredentialsProviderChain +AWSCredentialsProvider = DefaultCredentialsProvider # Appended to the user agent of the KCL. Does not impact the functionality of the # KCL in any other way. From e6402773d6e918718998b33bf8a9eef0441ae0ee Mon Sep 17 00:00:00 2001 From: Ethan Katnic Date: Wed, 21 Aug 2024 09:18:17 -0700 Subject: [PATCH 06/16] Fix syntax error, update method header for new clazz param --- .../config/AwsCredentialsProviderPropertyValueDecoder.java | 7 +++---- .../multilang/config/MultiLangDaemonConfiguration.java | 3 ++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AwsCredentialsProviderPropertyValueDecoder.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AwsCredentialsProviderPropertyValueDecoder.java index 536c0b1b..2246d057 100644 --- a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AwsCredentialsProviderPropertyValueDecoder.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AwsCredentialsProviderPropertyValueDecoder.java @@ -171,8 +171,9 @@ class AwsCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode * Attempts to construct an {@link AwsCredentialsProvider}. * * @param providerName Raw, unmodified provider name. Should there be an - * Exeception during construction, this parameter will be logged. + * Exception during construction, this parameter will be logged. * @param constructor supplier-like function that will perform the construction + * @param clazz the class to attempt to construct * @return the constructed provider, if successful; otherwise, null * * @param type of the CredentialsProvider to construct @@ -192,9 +193,7 @@ class AwsCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode } else { log.warn("Found non-static create() method in {}", providerName); } - } catch (NoSuchMethodException ignored) { - // Ignore if create() method is not found - } catch (IllegalAccessException | InvocationTargetException e1) { + } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e1) { log.warn("Exception thrown by create() method in {}", providerName, e1.getCause()); } } catch (NoSuchMethodException ignored) { diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/MultiLangDaemonConfiguration.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/MultiLangDaemonConfiguration.java index 54d9ac5c..5ee55a80 100644 --- a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/MultiLangDaemonConfiguration.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/MultiLangDaemonConfiguration.java @@ -280,7 +280,8 @@ public class MultiLangDaemonConfiguration { ArrayConverter arrayConverter = new ArrayConverter(String[].class, new StringConverter()); arrayConverter.setDelimiter(','); - convertUtilsBean.register(arrayConverter, String[].class);AwsCredentialsProviderPropertyValueDecoder credentialsDecoder = + convertUtilsBean.register(arrayConverter, String[].class); + AwsCredentialsProviderPropertyValueDecoder credentialsDecoder = new AwsCredentialsProviderPropertyValueDecoder(); Function converter = credentialsDecoder::decodeValue; From 4edccf42f05b1c43dd9946fdbbcae59fb368f87d Mon Sep 17 00:00:00 2001 From: Ethan Katnic Date: Wed, 21 Aug 2024 10:44:18 -0700 Subject: [PATCH 07/16] Add region validation to region setting in NestedPropertyProcessor --- .../amazon/kinesis/multilang/NestedPropertyKey.java | 10 ++++++++-- .../kinesis/multilang/NestedPropertyKeyTest.java | 5 +---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/NestedPropertyKey.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/NestedPropertyKey.java index 9902d28c..13acfeb1 100644 --- a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/NestedPropertyKey.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/NestedPropertyKey.java @@ -15,6 +15,7 @@ package software.amazon.kinesis.multilang; import java.util.HashMap; +import java.util.List; import java.util.Map; import com.google.common.base.CaseFormat; @@ -73,8 +74,13 @@ public enum NestedPropertyKey { * @see Available Regions */ ENDPOINT_REGION { - void visit(final NestedPropertyProcessor processor, final String region) { - processor.acceptEndpointRegion(Region.of(region)); + void visit(final NestedPropertyProcessor processor, final String regionName) { + List validRegions = Region.regions(); + Region region = Region.of(regionName); + if (!validRegions.contains(region)) { + throw new IllegalArgumentException("Invalid region name: " + regionName); + } + processor.acceptEndpointRegion(region); } }, diff --git a/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/NestedPropertyKeyTest.java b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/NestedPropertyKeyTest.java index 350b9fac..3c2de9c9 100644 --- a/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/NestedPropertyKeyTest.java +++ b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/NestedPropertyKeyTest.java @@ -70,10 +70,7 @@ public class NestedPropertyKeyTest { verify(mockProcessor).acceptEndpointRegion(expectedRegion); } - // Region.of(), which is invoked in this test, no longer throws an IllegalArgumentException when given an invalid - // region - // We would need to implement our own region validation to maintain this test - // @Test(expected = IllegalArgumentException.class) + @Test(expected = IllegalArgumentException.class) public void testInvalidEndpointRegion() { parse(mockProcessor, createKey(ENDPOINT_REGION, "snuffleupagus")); } From 8aa81dc255fa6c87c512e78b6f634bf4987e8060 Mon Sep 17 00:00:00 2001 From: Ethan Katnic Date: Wed, 21 Aug 2024 11:35:28 -0700 Subject: [PATCH 08/16] Remove unused old sdk version --- amazon-kinesis-client-multilang/pom.xml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/amazon-kinesis-client-multilang/pom.xml b/amazon-kinesis-client-multilang/pom.xml index 17663f32..9bbe79f1 100644 --- a/amazon-kinesis-client-multilang/pom.xml +++ b/amazon-kinesis-client-multilang/pom.xml @@ -27,10 +27,6 @@ amazon-kinesis-client-multilang - - 1.12.668 - - software.amazon.kinesis From f8eafb457a216fdb5833afcdd955dbc37c143764 Mon Sep 17 00:00:00 2001 From: Ethan Katnic Date: Thu, 22 Aug 2024 10:33:07 -0700 Subject: [PATCH 09/16] Add casting check for create() provider --- .../AwsCredentialsProviderPropertyValueDecoder.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AwsCredentialsProviderPropertyValueDecoder.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AwsCredentialsProviderPropertyValueDecoder.java index 2246d057..1e94ab56 100644 --- a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AwsCredentialsProviderPropertyValueDecoder.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AwsCredentialsProviderPropertyValueDecoder.java @@ -187,9 +187,16 @@ class AwsCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode return constructor.construct(); } catch (InstantiationException e) { try { + // Try to create an instance using .create() Method createMethod = clazz.getDeclaredMethod("create"); if (Modifier.isStatic(createMethod.getModifiers())) { - return (T) createMethod.invoke(null); + Object provider = createMethod.invoke(null); + if (provider instanceof AwsCredentialsProvider) { + return (T) provider; + } else { + log.warn("Returned provider is not an instance of {}", AwsCredentialsProvider.class.getName()); + return null; + } } else { log.warn("Found non-static create() method in {}", providerName); } From 6799c81f82be78d4aec5952a91b54d88356e5b03 Mon Sep 17 00:00:00 2001 From: Ethan Katnic Date: Tue, 27 Aug 2024 09:06:07 -0700 Subject: [PATCH 10/16] Simplify StsAssumeRoleCredentialsProvider logic --- .../KclStsAssumeRoleCredentialsProvider.java | 37 +++++++++-------- ...tsAssumeRoleCredentialsProviderConfig.java | 24 ----------- ...sAssumeRoleCredentialsProviderFactory.java | 40 ------------------- 3 files changed, 21 insertions(+), 80 deletions(-) delete mode 100644 amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/StsAssumeRoleCredentialsProviderConfig.java delete mode 100644 amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/StsAssumeRoleCredentialsProviderFactory.java diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/KclStsAssumeRoleCredentialsProvider.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/KclStsAssumeRoleCredentialsProvider.java index d077e6f2..d151fe7e 100644 --- a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/KclStsAssumeRoleCredentialsProvider.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/KclStsAssumeRoleCredentialsProvider.java @@ -1,53 +1,58 @@ package software.amazon.kinesis.multilang.auth; +import java.net.URI; import java.util.Arrays; import software.amazon.awssdk.auth.credentials.AwsCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.StsClientBuilder; import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider; +import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; +import software.amazon.awssdk.services.sts.model.AssumeRoleRequest.Builder; import software.amazon.kinesis.multilang.NestedPropertyKey; import software.amazon.kinesis.multilang.NestedPropertyProcessor; public class KclStsAssumeRoleCredentialsProvider implements AwsCredentialsProvider, NestedPropertyProcessor { - private final String roleArn; - private final String roleSessionName; - private Region region; - private String serviceEndpoint; - private String externalId; + private final Builder assumeRoleRequestBuilder; + private final StsClientBuilder stsClientBuilder; public KclStsAssumeRoleCredentialsProvider(String[] params) { this(params[0], params[1], Arrays.copyOfRange(params, 2, params.length)); } public KclStsAssumeRoleCredentialsProvider(String roleArn, String roleSessionName, String... params) { - this.roleArn = roleArn; - this.roleSessionName = roleSessionName; + this.assumeRoleRequestBuilder = + AssumeRoleRequest.builder().roleArn(roleArn).roleSessionName(roleSessionName); + this.stsClientBuilder = StsClient.builder(); NestedPropertyKey.parse(this, params); } @Override public AwsCredentials resolveCredentials() { - StsAssumeRoleCredentialsProviderConfig config = new StsAssumeRoleCredentialsProviderConfig( - roleArn, roleSessionName, region, serviceEndpoint, externalId); - StsAssumeRoleCredentialsProvider stsAssumeRoleCredentialsProvider = - StsAssumeRoleCredentialsProviderFactory.createProvider(config); - return stsAssumeRoleCredentialsProvider.resolveCredentials(); + StsClient stsClient = this.stsClientBuilder.build(); + AssumeRoleRequest assumeRoleRequest = this.assumeRoleRequestBuilder.build(); + StsAssumeRoleCredentialsProvider provider = StsAssumeRoleCredentialsProvider.builder() + .refreshRequest(assumeRoleRequest) + .stsClient(stsClient) + .build(); + return provider.resolveCredentials(); } @Override public void acceptEndpoint(String serviceEndpoint, String signingRegion) { - this.serviceEndpoint = serviceEndpoint; - this.region = Region.of(signingRegion); + stsClientBuilder.endpointOverride(URI.create(serviceEndpoint)); + stsClientBuilder.region(Region.of(signingRegion)); } @Override public void acceptEndpointRegion(Region region) { - this.region = region; + stsClientBuilder.region(region); } @Override public void acceptExternalId(String externalId) { - this.externalId = externalId; + assumeRoleRequestBuilder.externalId(externalId); } } diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/StsAssumeRoleCredentialsProviderConfig.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/StsAssumeRoleCredentialsProviderConfig.java deleted file mode 100644 index f4e41257..00000000 --- a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/StsAssumeRoleCredentialsProviderConfig.java +++ /dev/null @@ -1,24 +0,0 @@ -package software.amazon.kinesis.multilang.auth; - -import lombok.Getter; -import lombok.Setter; -import software.amazon.awssdk.regions.Region; - -@Getter -@Setter -public class StsAssumeRoleCredentialsProviderConfig { - private final String roleArn; - private final String roleSessionName; - private final Region region; - private final String serviceEndpoint; - private final String externalId; - - public StsAssumeRoleCredentialsProviderConfig( - String roleArn, String roleSessionName, Region region, String serviceEndpoint, String externalId) { - this.roleArn = roleArn; - this.roleSessionName = roleSessionName; - this.region = region; - this.serviceEndpoint = serviceEndpoint; - this.externalId = externalId; - } -} diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/StsAssumeRoleCredentialsProviderFactory.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/StsAssumeRoleCredentialsProviderFactory.java deleted file mode 100644 index b3b55531..00000000 --- a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/StsAssumeRoleCredentialsProviderFactory.java +++ /dev/null @@ -1,40 +0,0 @@ -package software.amazon.kinesis.multilang.auth; - -import java.net.URI; -import java.net.URISyntaxException; - -import software.amazon.awssdk.services.sts.StsClient; -import software.amazon.awssdk.services.sts.StsClientBuilder; -import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider; -import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; - -public class StsAssumeRoleCredentialsProviderFactory { - - public static StsAssumeRoleCredentialsProvider createProvider(StsAssumeRoleCredentialsProviderConfig config) { - StsClientBuilder stsClientBuilder = StsClient.builder(); - - if (config.getRegion() != null) { - stsClientBuilder.region(config.getRegion()); - } - - if (config.getServiceEndpoint() != null) { - try { - stsClientBuilder.endpointOverride(new URI(config.getServiceEndpoint())); - } catch (URISyntaxException e) { - throw new IllegalArgumentException("Invalid service endpoint: " + config.getServiceEndpoint(), e); - } - } - - StsClient stsClient = stsClientBuilder.build(); - - AssumeRoleRequest assumeRoleRequest = AssumeRoleRequest.builder() - .roleArn(config.getRoleArn()) - .roleSessionName(config.getRoleSessionName()) - .build(); - - return StsAssumeRoleCredentialsProvider.builder() - .refreshRequest(assumeRoleRequest) - .stsClient(stsClient) - .build(); - } -} From 22c2a7c91032e009da7652a59040619e98105378 Mon Sep 17 00:00:00 2001 From: Ethan Katnic Date: Tue, 27 Aug 2024 09:10:49 -0700 Subject: [PATCH 11/16] Simplify syntax of StsAssumeRoleCredentialsProvider --- .../multilang/auth/KclStsAssumeRoleCredentialsProvider.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/KclStsAssumeRoleCredentialsProvider.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/KclStsAssumeRoleCredentialsProvider.java index d151fe7e..ce9de0f5 100644 --- a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/KclStsAssumeRoleCredentialsProvider.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/KclStsAssumeRoleCredentialsProvider.java @@ -31,11 +31,9 @@ public class KclStsAssumeRoleCredentialsProvider implements AwsCredentialsProvid @Override public AwsCredentials resolveCredentials() { - StsClient stsClient = this.stsClientBuilder.build(); - AssumeRoleRequest assumeRoleRequest = this.assumeRoleRequestBuilder.build(); StsAssumeRoleCredentialsProvider provider = StsAssumeRoleCredentialsProvider.builder() - .refreshRequest(assumeRoleRequest) - .stsClient(stsClient) + .refreshRequest(assumeRoleRequestBuilder.build()) + .stsClient(stsClientBuilder.build()) .build(); return provider.resolveCredentials(); } From 444a4ddf9aa984865db51c702dc78e505d2954f7 Mon Sep 17 00:00:00 2001 From: Ethan Katnic Date: Tue, 27 Aug 2024 10:19:27 -0700 Subject: [PATCH 12/16] Add https check --- .../multilang/auth/KclStsAssumeRoleCredentialsProvider.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/KclStsAssumeRoleCredentialsProvider.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/KclStsAssumeRoleCredentialsProvider.java index ce9de0f5..65c096ae 100644 --- a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/KclStsAssumeRoleCredentialsProvider.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/KclStsAssumeRoleCredentialsProvider.java @@ -40,6 +40,9 @@ public class KclStsAssumeRoleCredentialsProvider implements AwsCredentialsProvid @Override public void acceptEndpoint(String serviceEndpoint, String signingRegion) { + if (!serviceEndpoint.startsWith("http://") && !serviceEndpoint.startsWith("https://")) { + serviceEndpoint = "https://" + serviceEndpoint; + } stsClientBuilder.endpointOverride(URI.create(serviceEndpoint)); stsClientBuilder.region(Region.of(signingRegion)); } From 921781b9f95004091a9b0684f2c3eab03d8ba084 Mon Sep 17 00:00:00 2001 From: Ethan Katnic Date: Thu, 29 Aug 2024 15:46:42 -0700 Subject: [PATCH 13/16] Simplify decoder logic for empty create constructor. Update documentation. --- ...edentialsProviderPropertyValueDecoder.java | 75 ++++++++----------- .../src/test/resources/multilang.properties | 2 +- .../configuring-credential-providers.md | 50 +++++++------ 3 files changed, 60 insertions(+), 67 deletions(-) diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AwsCredentialsProviderPropertyValueDecoder.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AwsCredentialsProviderPropertyValueDecoder.java index 1e94ab56..9a3f4599 100644 --- a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AwsCredentialsProviderPropertyValueDecoder.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AwsCredentialsProviderPropertyValueDecoder.java @@ -99,27 +99,34 @@ class AwsCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode final String[] varargs = Arrays.copyOfRange(nameAndArgs, 1, nameAndArgs.length); // attempt to invoke an explicit N-arg constructor of FooClass(String, String, ...) - provider = constructProvider( - providerName, - () -> { - Class[] argTypes = new Class[nameAndArgs.length - 1]; - Arrays.fill(argTypes, String.class); - return clazz.getConstructor(argTypes).newInstance(varargs); - }, - clazz); + provider = constructProvider(providerName, () -> { + Class[] argTypes = new Class[nameAndArgs.length - 1]; + Arrays.fill(argTypes, String.class); + return clazz.getConstructor(argTypes).newInstance(varargs); + }); if (provider == null) { // attempt to invoke a public varargs/array constructor of FooClass(String[]) - provider = constructProvider( - providerName, - () -> clazz.getConstructor(String[].class).newInstance((Object) varargs), - clazz); + provider = constructProvider(providerName, () -> clazz.getConstructor(String[].class) + .newInstance((Object) varargs)); } } if (provider == null) { - // regardless of parameters, fallback to invoke a public no-arg constructor - provider = constructProvider(providerName, clazz::newInstance, clazz); + // fallback to invoke a public no-arg constructor + provider = constructProvider(providerName, clazz::newInstance); + } + + if (provider == null) { + // if still not found, try empty create() method + try { + Method createMethod = clazz.getDeclaredMethod("create"); + if (Modifier.isStatic(createMethod.getModifiers())) { + provider = constructProvider(providerName, () -> clazz.cast(createMethod.invoke(null))); + } + } catch (NoSuchMethodException e) { + // No create() method found for class + } } if (provider != null) { @@ -145,17 +152,18 @@ class AwsCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode private static List getPossibleFullClassNames(final String provider) { return Stream.of( - // Customer provides a short name of common providers in com.amazonaws.auth package - // (e.g., any classes implementing the AWSCredentialsProvider interface) - // @see - // http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/AWSCredentialsProvider.html - "com.amazonaws.auth.", - // Customer provides a short name of a provider offered by this multi-lang package "software.amazon.kinesis.multilang.auth.", + // Customer provides a short name of common providers in software.amazon.awssdk.auth.credentials + // package (e.g., any classes implementing the AWSCredentialsProvider interface) + // @see + // https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/auth/credentials/AwsCredentialsProvider.html "software.amazon.awssdk.auth.credentials.", // Customer provides a fully-qualified provider name, or a custom credentials provider - // (e.g., com.amazonaws.auth.ClasspathFileCredentialsProvider, org.mycompany.FooProvider) + // (e.g., software.amazon.awssdk.auth.credentials.AwsCredentialsProvider) + + // Sts auth provider + "software.amazon.awssdk.services.sts.auth.", "") .map(prefix -> prefix + provider) .collect(Collectors.toList()); @@ -173,39 +181,18 @@ class AwsCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode * @param providerName Raw, unmodified provider name. Should there be an * Exception during construction, this parameter will be logged. * @param constructor supplier-like function that will perform the construction - * @param clazz the class to attempt to construct * @return the constructed provider, if successful; otherwise, null * * @param type of the CredentialsProvider to construct */ private static T constructProvider( - final String providerName, - final CredentialsProviderConstructor constructor, - Class clazz) { + final String providerName, final CredentialsProviderConstructor constructor) { try { // Try to create an instance using the given constructor return constructor.construct(); - } catch (InstantiationException e) { - try { - // Try to create an instance using .create() - Method createMethod = clazz.getDeclaredMethod("create"); - if (Modifier.isStatic(createMethod.getModifiers())) { - Object provider = createMethod.invoke(null); - if (provider instanceof AwsCredentialsProvider) { - return (T) provider; - } else { - log.warn("Returned provider is not an instance of {}", AwsCredentialsProvider.class.getName()); - return null; - } - } else { - log.warn("Found non-static create() method in {}", providerName); - } - } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e1) { - log.warn("Exception thrown by create() method in {}", providerName, e1.getCause()); - } } catch (NoSuchMethodException ignored) { // ignore - } catch (IllegalAccessException | InvocationTargetException | RuntimeException e) { + } catch (IllegalAccessException | InvocationTargetException | InstantiationException | RuntimeException e) { log.warn("Failed to construct {}", providerName, e); } return null; diff --git a/amazon-kinesis-client-multilang/src/test/resources/multilang.properties b/amazon-kinesis-client-multilang/src/test/resources/multilang.properties index 347d4e8a..067afcd8 100644 --- a/amazon-kinesis-client-multilang/src/test/resources/multilang.properties +++ b/amazon-kinesis-client-multilang/src/test/resources/multilang.properties @@ -19,7 +19,7 @@ applicationName = MultiLangTest # Users can change the credentials provider the KCL will use to retrieve credentials. # The DefaultAWSCredentialsProviderChain checks several other providers, which is # described here: -# http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html +# https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/auth/credentials/DefaultCredentialsProvider.html AWSCredentialsProvider = DefaultCredentialsProvider # Appended to the user agent of the KCL. Does not impact the functionality of the diff --git a/docs/multilang/configuring-credential-providers.md b/docs/multilang/configuring-credential-providers.md index 9b85baaa..c8401d9e 100644 --- a/docs/multilang/configuring-credential-providers.md +++ b/docs/multilang/configuring-credential-providers.md @@ -7,21 +7,22 @@ However, KCL now provides better extensibility to handle, and be enhanced to han This document should help multilang customers configure a suitable `CredentialProvider` (or contribute changes to support a new use case!). ## Sample Provider Configuration +DEPRECATED: StsAssumeRoleCredentialsProvider can no longer be constructed in this way: +``` +AWSCredentialsProvider = StsAssumeRoleCredentialsProvider||` +``` -In a Properties file, an `AWSCredentialsProperty` configuration might look like: -``` -AWSCredentialsProvider = STSAssumeRoleSessionCredentialsProvider|| -``` -This basic configuration creates an [STSAssumeRoleSessionCredentialsProvider][sts-assume-provider] with an ARN and session name. -While functional, this configuration is limited. -For example, this configuration cannot set a regional endpoint (e.g., VPC use case). +To create a [StsAssumeRoleCredentialsProvider][sts-assume-provider], see KclStsAssumeRoleCredentialsProvider below. -Leveraging nested properties, an `AWSCredentialsProperty` value might change to: +You can create a default [DefaultCredentialsProvider][default-credentials-provider] or [AnonymousCredentialsProvider][anonymous-credentials-provider] +by passing it in the config like: ``` -AWSCredentialsProvider = KclSTSAssumeRoleSessionCredentialsProvider||\ - |endpointRegion=us-east-1|externalId=spartacus +AWSCredentialsProvider = DefaultCredentialsProvider ``` -N.B. Backslash (`\`) is for multi-line legibility and is not required. + +If you wish to customize properties on an AWS SDK provider that uses a builder, like the StsASsumeRoleCredentialsProvider, +you will need to wrap this provider class, provide a constructor, and manage the build of the provider. +See implementation of [KclStsAssumeRoleCredentialsProvider][kcl-sts-provider] ## Nested Properties @@ -36,9 +37,6 @@ The [Backus-Naur form][bnf] of the value: # this depends on the nested key ``` -In general, required parameters are passed directly to the class' constructor -(e.g., [STSAssumeRoleSessionCredentialsProvider(String, String)][sts-assume-provider-constructor]). - Nested properties are a custom mapping provided by KCL multilang, and do not exist in the AWS SDK. See [NestedPropertyKey][nested-property-key] for the supported keys, and details on their expected values. @@ -54,18 +52,26 @@ A backwards-compatible addition might look like: } ``` -### KclSTSAssumeRoleSessionCredentialsProvider - -KCL multilang includes a [custom nested property processor for `STSAssumeRole`][kcl-sts-provider]. -Multilang configurations that use `STSAssumeRoleSessionCredentialsProvider` need only prefix `Kcl` to exercise this new provider: +Leveraging nested properties, an `AWSCredentialsProperty` value might look like: ``` -AWSCredentialsProvider = KclSTSAssumeRoleSessionCredentialsProvider|| +AWSCredentialsProvider = KclSTSAssumeRoleSessionCredentialsProvider||\ + |endpointRegion=us-east-1|externalId=spartacus +``` + +N.B. Backslash (`\`) is for multi-line legibility and is not required. +### KclStsAssumeRoleCredentialsProvider + +KCL multilang includes a [custom nested property processor for `StsAssumeRole`][kcl-sts-provider]. +Multilang configurations that use `StsAssumeRoleSessionCredentialsProvider` need only prefix `Kcl` to exercise this new provider: +``` +AWSCredentialsProvider = KclStsAssumeRoleCredentialsProvider|| ``` [aws-credentials-provider]: https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/auth/credentials/AwsCredentialsProvider.html [bnf]: https://en.wikipedia.org/wiki/Backus%E2%80%93Naur_form -[kcl-sts-provider]: /amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/KclSTSAssumeRoleSessionCredentialsProvider.java +[kcl-sts-provider]: /amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/KclStsAssumeRoleCredentialsProvider.java [nested-property-key]: /amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/NestedPropertyKey.java [nested-property-processor]: /amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/NestedPropertyProcessor.java -[sts-assume-provider]: https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/STSAssumeRoleSessionCredentialsProvider.html -[sts-assume-provider-constructor]: https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/STSAssumeRoleSessionCredentialsProvider.html#STSAssumeRoleSessionCredentialsProvider-java.lang.String-java.lang.String- +[sts-assume-provider]: https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/sts/auth/StsAssumeRoleCredentialsProvider.html +[default-credentials-provider]: https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/auth/credentials/DefaultCredentialsProvider.html +[anonymous-credentials-provider]: https://sdk.amazonaws.com/java/api/2.0.0-preview-11/software/amazon/awssdk/auth/credentials/AnonymousCredentialsProvider.html From 9b342180e0c6d215be74dd27fe5ebf51deb5b0de Mon Sep 17 00:00:00 2001 From: Ethan Katnic Date: Thu, 29 Aug 2024 16:13:02 -0700 Subject: [PATCH 14/16] Clean up unused changes --- .../config/AwsCredentialsProviderPropertyValueDecoder.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AwsCredentialsProviderPropertyValueDecoder.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AwsCredentialsProviderPropertyValueDecoder.java index 9a3f4599..4cdd702e 100644 --- a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AwsCredentialsProviderPropertyValueDecoder.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AwsCredentialsProviderPropertyValueDecoder.java @@ -113,7 +113,7 @@ class AwsCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode } if (provider == null) { - // fallback to invoke a public no-arg constructor + // regardless of parameters, fallback to invoke a public no-arg constructor provider = constructProvider(providerName, clazz::newInstance); } @@ -123,6 +123,8 @@ class AwsCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode Method createMethod = clazz.getDeclaredMethod("create"); if (Modifier.isStatic(createMethod.getModifiers())) { provider = constructProvider(providerName, () -> clazz.cast(createMethod.invoke(null))); + } else { + log.warn("Found non-static create() method in {}", providerName); } } catch (NoSuchMethodException e) { // No create() method found for class @@ -188,11 +190,10 @@ class AwsCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode private static T constructProvider( final String providerName, final CredentialsProviderConstructor constructor) { try { - // Try to create an instance using the given constructor return constructor.construct(); } catch (NoSuchMethodException ignored) { // ignore - } catch (IllegalAccessException | InvocationTargetException | InstantiationException | RuntimeException e) { + } catch (IllegalAccessException | InstantiationException | InvocationTargetException | RuntimeException e) { log.warn("Failed to construct {}", providerName, e); } return null; From 6deff859945e7e57e716e51b2d7b34d603d45ae1 Mon Sep 17 00:00:00 2001 From: Ethan Katnic Date: Thu, 29 Aug 2024 16:16:18 -0700 Subject: [PATCH 15/16] Remove unused prefix --- .../config/AwsCredentialsProviderPropertyValueDecoder.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AwsCredentialsProviderPropertyValueDecoder.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AwsCredentialsProviderPropertyValueDecoder.java index 4cdd702e..cda7077d 100644 --- a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AwsCredentialsProviderPropertyValueDecoder.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AwsCredentialsProviderPropertyValueDecoder.java @@ -163,9 +163,6 @@ class AwsCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode "software.amazon.awssdk.auth.credentials.", // Customer provides a fully-qualified provider name, or a custom credentials provider // (e.g., software.amazon.awssdk.auth.credentials.AwsCredentialsProvider) - - // Sts auth provider - "software.amazon.awssdk.services.sts.auth.", "") .map(prefix -> prefix + provider) .collect(Collectors.toList()); From 77a3440f0bb4d61eadbba29f7bd6852eb29a7461 Mon Sep 17 00:00:00 2001 From: Ethan Katnic Date: Thu, 29 Aug 2024 16:21:23 -0700 Subject: [PATCH 16/16] Clean up legacy library names --- .../AwsCredentialsProviderPropertyValueDecoder.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AwsCredentialsProviderPropertyValueDecoder.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AwsCredentialsProviderPropertyValueDecoder.java index cda7077d..112fb52c 100644 --- a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AwsCredentialsProviderPropertyValueDecoder.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AwsCredentialsProviderPropertyValueDecoder.java @@ -29,7 +29,7 @@ import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain; /** - * Get AWSCredentialsProvider property. + * Get AwsCredentialsProvider property. */ @Slf4j class AwsCredentialsProviderPropertyValueDecoder implements IPropertyValueDecoder { @@ -42,7 +42,7 @@ class AwsCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode AwsCredentialsProviderPropertyValueDecoder() {} /** - * Get AWSCredentialsProvider property. + * Get AwsCredentialsProvider property. * * @param value * property value as String @@ -59,7 +59,7 @@ class AwsCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode .credentialsProviders(providers) .build(); } else { - throw new IllegalArgumentException("Property AWSCredentialsProvider is missing."); + throw new IllegalArgumentException("Property AwsCredentialsProvider is missing."); } } @@ -157,12 +157,12 @@ class AwsCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode // Customer provides a short name of a provider offered by this multi-lang package "software.amazon.kinesis.multilang.auth.", // Customer provides a short name of common providers in software.amazon.awssdk.auth.credentials - // package (e.g., any classes implementing the AWSCredentialsProvider interface) + // package (e.g., any classes implementing the AwsCredentialsProvider interface) // @see // https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/auth/credentials/AwsCredentialsProvider.html "software.amazon.awssdk.auth.credentials.", // Customer provides a fully-qualified provider name, or a custom credentials provider - // (e.g., software.amazon.awssdk.auth.credentials.AwsCredentialsProvider) + // (e.g., org.mycompany.FooProvider) "") .map(prefix -> prefix + provider) .collect(Collectors.toList());