From 03c15eb2751a3e2510e01ef4ea44c2038bd4c6e9 Mon Sep 17 00:00:00 2001 From: Sahil Palvia Date: Mon, 14 Jan 2019 17:35:35 -0800 Subject: [PATCH] Introducing MultiLangDaemon support: (#483) * Introducing MultiLangDaemon support for Enhanced Fan-Out. * MultiLangDaemon now supports the following command line options. * `--properties-file`: Properties file that the KCL should use to set up the Scheduler. * `--log-configuration`: logback.xml that the KCL should use for logging. * Updated AWS SDK dependency to 2.2.0. * MultiLangDaemon now uses logback for logging. --- CHANGELOG.md | 9 + README.md | 18 +- amazon-kinesis-client-multilang/pom.xml | 59 +- .../kinesis/multilang/MultiLangDaemon.java | 168 ---- .../config/BooleanPropertyValueDecoder.java | 48 - ...lPositionInStreamPropertyValueDecoder.java | 49 - .../config/KinesisClientLibConfigurator.java | 225 ----- .../config/LongPropertyValueDecoder.java | 48 - .../config/SetPropertyValueDecoder.java | 65 -- .../config/StringPropertyValueDecoder.java | 53 -- .../multilang/messages/StatusMessage.java | 40 - .../multilang/DrainChildSTDERRTask.java | 20 +- .../multilang/DrainChildSTDOUTTask.java | 2 +- .../kinesis/multilang/GetNextMessageTask.java | 4 +- .../kinesis/multilang/LineReaderTask.java | 2 +- .../kinesis/multilang/MessageReader.java | 22 +- .../kinesis/multilang/MessageWriter.java | 39 +- .../kinesis/multilang/MultiLangDaemon.java | 234 +++++ .../multilang/MultiLangDaemonConfig.java | 26 +- .../kinesis/multilang/MultiLangProtocol.java | 81 +- .../MultiLangRecordProcessorFactory.java | 9 +- .../MultiLangShardRecordProcessor.java | 33 +- ...edentialsProviderPropertyValueDecoder.java | 26 +- .../multilang/config/BuilderDynaBean.java | 286 ++++++ .../config/ConfigurationSettable.java | 49 + .../config/ConfigurationSettableUtils.java | 109 +++ .../config/ConfigurationSettables.java | 27 + .../config/DatePropertyValueDecoder.java | 2 +- .../config/DynaBeanBuilderSupport.java | 253 ++++++ .../config/DynaBeanBuilderUtils.java | 56 ++ .../config/DynaBeanCreateSupport.java | 101 +++ .../multilang/config/FanoutConfigBean.java | 46 + .../config/IPropertyValueDecoder.java | 2 +- .../config/IntegerPropertyValueDecoder.java | 2 +- .../config/KinesisClientLibConfigurator.java | 100 +++ .../config/MultiLangDaemonConfiguration.java | 394 ++++++++ .../multilang/config/PollingConfigBean.java | 63 ++ .../config/RetrievalConfigBuilder.java | 32 + .../multilang/config/RetrievalMode.java | 64 ++ .../kinesis/multilang/config/TypeTag.java | 28 + .../credentials/V2CredentialWrapper.java | 49 + .../multilang/messages/CheckpointMessage.java | 3 +- .../multilang/messages/InitializeMessage.java | 2 +- .../messages/JsonFriendlyRecord.java | 21 +- .../multilang/messages/LeaseLostMessage.java | 25 + .../kinesis/multilang/messages/Message.java | 6 +- .../messages/ProcessRecordsMessage.java | 2 +- .../multilang/messages/ShardEndedMessage.java | 23 + .../multilang/messages/ShutdownMessage.java | 21 +- .../messages/ShutdownRequestedMessage.java | 2 +- .../multilang/messages/StatusMessage.java | 39 + .../kinesis/multilang/package-info.java | 57 +- .../multilang/MultiLangDaemonTest.java | 51 -- ...tialsProviderPropertyValueDecoderTest.java | 103 --- .../amazon}/kinesis/multilang/Matchers.java | 2 +- .../kinesis/multilang/MessageReaderTest.java | 26 +- .../kinesis/multilang/MessageWriterTest.java | 55 +- .../multilang/MultiLangDaemonConfigTest.java | 26 +- .../multilang/MultiLangDaemonTest.java | 276 ++++++ .../multilang/MultiLangProtocolTest.java | 47 +- .../kinesis/multilang/ReadSTDERRTaskTest.java | 22 +- ...eamingShardRecordProcessorFactoryTest.java | 7 +- .../StreamingShardRecordProcessorTest.java | 101 ++- ...tialsProviderPropertyValueDecoderTest.java | 167 ++++ .../multilang/config/BuilderDynaBeanTest.java | 844 ++++++++++++++++++ .../ConfigurationSettableUtilsTest.java | 179 ++++ .../config/DatePropertyValueDecoderTest.java | 2 +- .../config/FanoutConfigBeanTest.java | 67 ++ .../KinesisClientLibConfiguratorTest.java | 190 ++-- .../MultiLangDaemonConfigurationTest.java | 177 ++++ .../config/PollingConfigBeanTest.java | 62 ++ .../messages/JsonFriendlyRecordTest.java | 167 ++++ .../multilang/messages/MessageTest.java | 13 +- .../{logback.xml => logback-test.xml} | 0 amazon-kinesis-client/pom.xml | 6 +- .../kinesis/common/KinesisClientUtil.java | 7 +- pom.xml | 4 +- 77 files changed, 4494 insertions(+), 1251 deletions(-) delete mode 100644 amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangDaemon.java delete mode 100644 amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/BooleanPropertyValueDecoder.java delete mode 100644 amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/InitialPositionInStreamPropertyValueDecoder.java delete mode 100644 amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/KinesisClientLibConfigurator.java delete mode 100644 amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/LongPropertyValueDecoder.java delete mode 100644 amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/SetPropertyValueDecoder.java delete mode 100644 amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/StringPropertyValueDecoder.java delete mode 100644 amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/messages/StatusMessage.java rename amazon-kinesis-client-multilang/src/main/java/{com/amazonaws/services => software/amazon}/kinesis/multilang/DrainChildSTDERRTask.java (60%) rename amazon-kinesis-client-multilang/src/main/java/{com/amazonaws/services => software/amazon}/kinesis/multilang/DrainChildSTDOUTTask.java (97%) rename amazon-kinesis-client-multilang/src/main/java/{com/amazonaws/services => software/amazon}/kinesis/multilang/GetNextMessageTask.java (96%) rename amazon-kinesis-client-multilang/src/main/java/{com/amazonaws/services => software/amazon}/kinesis/multilang/LineReaderTask.java (99%) rename amazon-kinesis-client-multilang/src/main/java/{com/amazonaws/services => software/amazon}/kinesis/multilang/MessageReader.java (87%) rename amazon-kinesis-client-multilang/src/main/java/{com/amazonaws/services => software/amazon}/kinesis/multilang/MessageWriter.java (82%) create mode 100644 amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MultiLangDaemon.java rename amazon-kinesis-client-multilang/src/main/java/{com/amazonaws/services => software/amazon}/kinesis/multilang/MultiLangDaemonConfig.java (89%) rename amazon-kinesis-client-multilang/src/main/java/{com/amazonaws/services => software/amazon}/kinesis/multilang/MultiLangProtocol.java (82%) rename amazon-kinesis-client-multilang/src/main/java/{com/amazonaws/services => software/amazon}/kinesis/multilang/MultiLangRecordProcessorFactory.java (89%) rename amazon-kinesis-client-multilang/src/main/java/{com/amazonaws/services => software/amazon}/kinesis/multilang/MultiLangShardRecordProcessor.java (93%) rename amazon-kinesis-client-multilang/src/main/java/{com/amazonaws/services => software/amazon}/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoder.java (83%) create mode 100644 amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/BuilderDynaBean.java create mode 100644 amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/ConfigurationSettable.java create mode 100644 amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/ConfigurationSettableUtils.java create mode 100644 amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/ConfigurationSettables.java rename amazon-kinesis-client-multilang/src/main/java/{com/amazonaws/services => software/amazon}/kinesis/multilang/config/DatePropertyValueDecoder.java (96%) create mode 100644 amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/DynaBeanBuilderSupport.java create mode 100644 amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/DynaBeanBuilderUtils.java create mode 100644 amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/DynaBeanCreateSupport.java create mode 100644 amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/FanoutConfigBean.java rename amazon-kinesis-client-multilang/src/main/java/{com/amazonaws/services => software/amazon}/kinesis/multilang/config/IPropertyValueDecoder.java (95%) rename amazon-kinesis-client-multilang/src/main/java/{com/amazonaws/services => software/amazon}/kinesis/multilang/config/IntegerPropertyValueDecoder.java (95%) create mode 100644 amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/KinesisClientLibConfigurator.java create mode 100644 amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/MultiLangDaemonConfiguration.java create mode 100644 amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/PollingConfigBean.java create mode 100644 amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/RetrievalConfigBuilder.java create mode 100644 amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/RetrievalMode.java create mode 100644 amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/TypeTag.java create mode 100644 amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/credentials/V2CredentialWrapper.java rename amazon-kinesis-client-multilang/src/main/java/{com/amazonaws/services => software/amazon}/kinesis/multilang/messages/CheckpointMessage.java (95%) rename amazon-kinesis-client-multilang/src/main/java/{com/amazonaws/services => software/amazon}/kinesis/multilang/messages/InitializeMessage.java (97%) rename amazon-kinesis-client-multilang/src/main/java/{com/amazonaws/services => software/amazon}/kinesis/multilang/messages/JsonFriendlyRecord.java (73%) create mode 100644 amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/LeaseLostMessage.java rename amazon-kinesis-client-multilang/src/main/java/{com/amazonaws/services => software/amazon}/kinesis/multilang/messages/Message.java (88%) rename amazon-kinesis-client-multilang/src/main/java/{com/amazonaws/services => software/amazon}/kinesis/multilang/messages/ProcessRecordsMessage.java (97%) create mode 100644 amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/ShardEndedMessage.java rename amazon-kinesis-client-multilang/src/main/java/{com/amazonaws/services => software/amazon}/kinesis/multilang/messages/ShutdownMessage.java (52%) rename amazon-kinesis-client-multilang/src/main/java/{com/amazonaws/services => software/amazon}/kinesis/multilang/messages/ShutdownRequestedMessage.java (93%) create mode 100644 amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/StatusMessage.java rename amazon-kinesis-client-multilang/src/main/java/{com/amazonaws/services => software/amazon}/kinesis/multilang/package-info.java (72%) delete mode 100644 amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/MultiLangDaemonTest.java delete mode 100644 amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoderTest.java rename amazon-kinesis-client-multilang/src/test/java/{com/amazonaws/services => software/amazon}/kinesis/multilang/Matchers.java (98%) rename amazon-kinesis-client-multilang/src/test/java/{com/amazonaws/services => software/amazon}/kinesis/multilang/MessageReaderTest.java (90%) rename amazon-kinesis-client-multilang/src/test/java/{com/amazonaws/services => software/amazon}/kinesis/multilang/MessageWriterTest.java (73%) rename amazon-kinesis-client-multilang/src/test/java/{com/amazonaws/services => software/amazon}/kinesis/multilang/MultiLangDaemonConfigTest.java (78%) create mode 100644 amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MultiLangDaemonTest.java rename amazon-kinesis-client-multilang/src/test/java/{com/amazonaws/services => software/amazon}/kinesis/multilang/MultiLangProtocolTest.java (85%) rename amazon-kinesis-client-multilang/src/test/java/{com/amazonaws/services => software/amazon}/kinesis/multilang/ReadSTDERRTaskTest.java (79%) rename amazon-kinesis-client-multilang/src/test/java/{com/amazonaws/services => software/amazon}/kinesis/multilang/StreamingShardRecordProcessorFactoryTest.java (81%) rename amazon-kinesis-client-multilang/src/test/java/{com/amazonaws/services => software/amazon}/kinesis/multilang/StreamingShardRecordProcessorTest.java (89%) create mode 100644 amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoderTest.java create mode 100644 amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/BuilderDynaBeanTest.java create mode 100644 amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/ConfigurationSettableUtilsTest.java rename amazon-kinesis-client-multilang/src/test/java/{com/amazonaws/services => software/amazon}/kinesis/multilang/config/DatePropertyValueDecoderTest.java (96%) create mode 100644 amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/FanoutConfigBeanTest.java rename amazon-kinesis-client-multilang/src/test/java/{com/amazonaws/services => software/amazon}/kinesis/multilang/config/KinesisClientLibConfiguratorTest.java (68%) create mode 100644 amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/MultiLangDaemonConfigurationTest.java create mode 100644 amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/PollingConfigBeanTest.java create mode 100644 amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/messages/JsonFriendlyRecordTest.java rename amazon-kinesis-client-multilang/src/test/java/{com/amazonaws/services => software/amazon}/kinesis/multilang/messages/MessageTest.java (83%) rename amazon-kinesis-client-multilang/src/test/resources/{logback.xml => logback-test.xml} (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index b22ef573..b31f80a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +### Release 2.1.0 (January 14, 2019) +[Milestone #27](https://github.com/awslabs/amazon-kinesis-client/milestone/27) +* Introducing MultiLangDaemon support for Enhanced Fan-Out. +* MultiLangDaemon now supports the following command line options. + * `--properties-file`: Properties file that the KCL should use to set up the Scheduler. + * `--log-configuration`: logback.xml that the KCL should use for logging. +* Updated AWS SDK dependency to 2.2.0. +* MultiLangDaemon now uses logback for logging. + ### Release 2.0.5 (November 12, 2018) [Milestone #26](https://github.com/awslabs/amazon-kinesis-client/milestone/26?closed=1) * Fixed a deadlock condition that could occur when using the polling model. diff --git a/README.md b/README.md index a1523497..efd7f5a0 100644 --- a/README.md +++ b/README.md @@ -61,14 +61,14 @@ The recommended way to use the KCL for Java is to consume it from Maven. ## Release Notes -### Latest Release (2.0.5 - November 12, 2018) -[Milestone #26](https://github.com/awslabs/amazon-kinesis-client/milestone/26?closed=1) -* Fixed a deadlock condition that could occur when using the polling model. - It was possible to hit a deadlock in the retrieval of records When using the `PollingConfig` and a slow running record processor. - * [PR #462](https://github.com/awslabs/amazon-kinesis-client/pull/462) - * [Issue #448](https://github.com/awslabs/amazon-kinesis-client/issues/448) -* Adjusted `RetrievalConfig`, and `FanOutConfig` to use accessors instead of direct member access. - * [PR #453](https://github.com/awslabs/amazon-kinesis-client/pull/453) +### Latest Release (2.1.0 - January 14, 2019) +[Milestone #27](https://github.com/awslabs/amazon-kinesis-client/milestone/27) +* Introducing MultiLangDaemon support for Enhanced Fan-Out. +* MultiLangDaemon now supports the following command line options. + * `--properties-file`: Properties file that the KCL should use to set up the Scheduler. + * `--log-configuration`: logback.xml that the KCL should use for logging. +* Updated AWS SDK dependency to 2.2.0. +* MultiLangDaemon now uses logback for logging. ### For remaining release notes check **[CHANGELOG.md][changelog-md]**. @@ -84,6 +84,6 @@ The recommended way to use the KCL for Java is to consume it from Maven. [kinesis-guide-kpl]: http://docs.aws.amazon.com//kinesis/latest/dev/developing-producers-with-kpl.html [kinesis-guide-consumer-deaggregation]: http://docs.aws.amazon.com//kinesis/latest/dev/kinesis-kpl-consumer-deaggregation.html [kclpy]: https://github.com/awslabs/amazon-kinesis-client-python -[multi-lang-protocol]: https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/multilang/package-info.java +[multi-lang-protocol]: https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/package-info.java [changelog-md]: https://github.com/awslabs/amazon-kinesis-client/blob/master/CHANGELOG.md [migration-guide]: https://docs.aws.amazon.com/streams/latest/dev/kcl-migration.html diff --git a/amazon-kinesis-client-multilang/pom.xml b/amazon-kinesis-client-multilang/pom.xml index 8897a8b6..6a7573d6 100644 --- a/amazon-kinesis-client-multilang/pom.xml +++ b/amazon-kinesis-client-multilang/pom.xml @@ -19,18 +19,47 @@ amazon-kinesis-client-pom software.amazon.kinesis - 2.0.5 + 2.1.0 4.0.0 amazon-kinesis-client-multilang + + 1.11.477 + + software.amazon.kinesis amazon-kinesis-client ${project.version} + + software.amazon.awssdk + sts + ${awssdk.version} + + + + com.amazonaws + aws-java-sdk-core + ${aws-java-sdk.version} + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.dataformat + jackson-dataformat-cbor + + + org.apache.httpcomponents + httpclient + + + org.projectlombok @@ -38,11 +67,32 @@ 1.16.20 provided - ch.qos.logback logback-classic - 1.1.7 + 1.2.3 + + + com.beust + jcommander + 1.72 + + + commons-io + commons-io + 2.6 + + + org.apache.commons + commons-collections4 + 4.2 + + + + + commons-beanutils + commons-beanutils + 1.9.3 @@ -52,21 +102,18 @@ 4.11 test - org.mockito mockito-all 1.10.19 test - org.hamcrest hamcrest-all 1.3 test - diff --git a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangDaemon.java b/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangDaemon.java deleted file mode 100644 index ecb70d22..00000000 --- a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangDaemon.java +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/asl/ - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ -package com.amazonaws.services.kinesis.multilang; - -import java.io.IOException; -import java.io.PrintStream; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import lombok.extern.slf4j.Slf4j; -import software.amazon.kinesis.coordinator.KinesisClientLibConfiguration; -import software.amazon.kinesis.coordinator.Scheduler; -import software.amazon.kinesis.processor.ShardRecordProcessorFactory; - -/** - * Main app that launches the scheduler that runs the multi-language record processor. - * - * Requires a properties file containing configuration for this daemon and the KCL. A properties file should at minimum - * define these properties: - * - * 
- * # The script that abides by the multi-language protocol. This script will
- * # be executed by the MultiLangDaemon, which will communicate with this script
- * # over STDIN and STDOUT according to the multi-language protocol.
- * executableName = sampleapp.py
- *
- * # The name of an Amazon Kinesis stream to process.
- * streamName = words
- *
- * # Used by the KCL as the name of this application. Will be used as the name
- * # of a Amazon DynamoDB table which will store the lease and checkpoint
- * # information for workers with this application name.
- * applicationName = PythonKCLSample
- *
- * # Users can change the credentials provider the KCL will use to retrieve credentials.
- * # The DefaultAWSCredentialsProviderChain checks several other providers, which is
- * # described here:
- * # http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html
- * AWSCredentialsProvider = DefaultAWSCredentialsProviderChain
- *
- */ -@Slf4j -public class MultiLangDaemon implements Callable { - private Scheduler scheduler; - - /** - * Constructor. - * - * @param configuration The KCL config to use. - * @param recordProcessorFactory A record processor factory to create record processors that abide by the multi-lang - * protocol. - * @param workerThreadPool The executor service to run the daemon in. - */ - public MultiLangDaemon(KinesisClientLibConfiguration configuration, - MultiLangRecordProcessorFactory recordProcessorFactory, - ExecutorService workerThreadPool) { - this(buildWorker(recordProcessorFactory, configuration, workerThreadPool)); - } - - private static Scheduler buildWorker(ShardRecordProcessorFactory recordShardRecordProcessorFactory, - KinesisClientLibConfiguration configuration, ExecutorService workerThreadPool) { - return null; - } - - /** - * - * @param scheduler A scheduler to use instead of the default scheduler. - */ - public MultiLangDaemon(Scheduler scheduler) { - this.scheduler = scheduler; - } - - /** - * Utility for describing how to run this app. - * - * @param stream Where to output the usage info. - * @param messageToPrepend An optional error message to describe why the usage is being printed. - */ - public static void printUsage(PrintStream stream, String messageToPrepend) { - StringBuilder builder = new StringBuilder(); - if (messageToPrepend != null) { - builder.append(messageToPrepend); - } - builder.append(String.format("java %s ", MultiLangDaemon.class.getCanonicalName())); - stream.println(builder.toString()); - } - - @Override - public Integer call() throws Exception { - int exitCode = 0; - try { - scheduler.run(); - } catch (Throwable t) { - log.error("Caught throwable while processing data.", t); - exitCode = 1; - } - return exitCode; - } - - /** - * @param args Accepts a single argument, that argument is a properties file which provides KCL configuration as - * well as the name of an executable. - */ - public static void main(String[] args) { - - if (args.length == 0) { - printUsage(System.err, "You must provide a properties file"); - System.exit(1); - } - MultiLangDaemonConfig config = null; - try { - config = new MultiLangDaemonConfig(args[0]); - } catch (IOException e) { - printUsage(System.err, "You must provide a properties file"); - System.exit(1); - } catch (IllegalArgumentException e) { - printUsage(System.err, e.getMessage()); - System.exit(1); - } - - ExecutorService executorService = config.getExecutorService(); - - // Daemon - final MultiLangDaemon daemon = new MultiLangDaemon( - config.getKinesisClientLibConfiguration(), - config.getRecordProcessorFactory(), - executorService); - - final long shutdownGraceMillis = config.getKinesisClientLibConfiguration().getShutdownGraceMillis(); - Runtime.getRuntime().addShutdownHook(new Thread() { - @Override - public void run() { - log.info("Process terminated, will initiate shutdown."); - try { - Future fut = daemon.scheduler.requestShutdown(); - fut.get(shutdownGraceMillis, TimeUnit.MILLISECONDS); - log.info("Process shutdown is complete."); - } catch (InterruptedException | ExecutionException | TimeoutException e) { - log.error("Encountered an error during shutdown.", e); - } - } - }); - - Future future = executorService.submit(daemon); - try { - System.exit(future.get()); - } catch (InterruptedException | ExecutionException e) { - log.error("Encountered an error while running daemon", e); - } - System.exit(1); - } -} diff --git a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/BooleanPropertyValueDecoder.java b/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/BooleanPropertyValueDecoder.java deleted file mode 100644 index e57413dd..00000000 --- a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/BooleanPropertyValueDecoder.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/asl/ - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ -package com.amazonaws.services.kinesis.multilang.config; - -import java.util.Arrays; -import java.util.List; - -/** - * Provide boolean property. - */ -class BooleanPropertyValueDecoder implements IPropertyValueDecoder { - - /** - * Constructor. - */ - BooleanPropertyValueDecoder() { - } - - /** - * @param value property value as String - * @return corresponding variable in correct type - */ - @Override - public Boolean decodeValue(String value) { - return Boolean.parseBoolean(value); - } - - /** - * @return list of supported types - */ - @Override - public List> getSupportedTypes() { - return Arrays.asList(boolean.class, Boolean.class); - } - -} diff --git a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/InitialPositionInStreamPropertyValueDecoder.java b/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/InitialPositionInStreamPropertyValueDecoder.java deleted file mode 100644 index 0b44273a..00000000 --- a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/InitialPositionInStreamPropertyValueDecoder.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/asl/ - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ -package com.amazonaws.services.kinesis.multilang.config; - -import java.util.Arrays; -import java.util.List; - -import software.amazon.kinesis.common.InitialPositionInStream; - -/** - * Get an InitialiPosition enum property. - */ -class InitialPositionInStreamPropertyValueDecoder implements IPropertyValueDecoder { - - /** - * Constructor. - */ - InitialPositionInStreamPropertyValueDecoder() { - } - - /** - * @param value property value as String - * @return corresponding variable in correct type - */ - @Override - public InitialPositionInStream decodeValue(String value) { - return InitialPositionInStream.valueOf(value.toUpperCase()); - } - - /** - * @return list of supported types - */ - @Override - public List> getSupportedTypes() { - return Arrays.asList(InitialPositionInStream.class); - } -} diff --git a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/KinesisClientLibConfigurator.java b/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/KinesisClientLibConfigurator.java deleted file mode 100644 index 853a7cc9..00000000 --- a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/KinesisClientLibConfigurator.java +++ /dev/null @@ -1,225 +0,0 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/asl/ - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ -package com.amazonaws.services.kinesis.multilang.config; - -import lombok.extern.slf4j.Slf4j; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.kinesis.coordinator.KinesisClientLibConfiguration; - -import java.io.IOException; -import java.io.InputStream; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Hashtable; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; -import java.util.UUID; - -/** - * KinesisClientLibConfigurator constructs a KinesisClientLibConfiguration from java properties file. The following - * three properties must be provided. 1) "applicationName" 2) "streamName" 3) "AWSCredentialsProvider" - * KinesisClientLibConfigurator will help to automatically assign the value of "workerId" if this property is not - * provided. In the specified properties file, any properties, which matches the variable name in - * KinesisClientLibConfiguration and has a corresponding "with{variableName}" setter method, will be read in, and its - * value in properties file will be assigned to corresponding variable in KinesisClientLibConfiguration. - */ -@Slf4j -public class KinesisClientLibConfigurator { - private static final String PREFIX = "with"; - - // Required properties - private static final String PROP_APP_NAME = "applicationName"; - private static final String PROP_STREAM_NAME = "streamName"; - private static final String PROP_CREDENTIALS_PROVIDER_KINESIS = "AWSCredentialsProvider"; - private static final String PROP_CREDENTIALS_PROVIDER_DYNAMODB = "AWSCredentialsProviderDynamoDB"; - private static final String PROP_CREDENTIALS_PROVIDER_CLOUDWATCH = "AWSCredentialsProviderCloudWatch"; - private static final String PROP_WORKER_ID = "workerId"; - - private Map, IPropertyValueDecoder> classToDecoder; - private Map> nameToMethods; - - /** - * Constructor. - */ - public KinesisClientLibConfigurator() { - List> getters = - Arrays.asList(new IntegerPropertyValueDecoder(), - new LongPropertyValueDecoder(), - new BooleanPropertyValueDecoder(), - new DatePropertyValueDecoder(), - new AWSCredentialsProviderPropertyValueDecoder(), - new StringPropertyValueDecoder(), - new InitialPositionInStreamPropertyValueDecoder(), - new SetPropertyValueDecoder()); - - classToDecoder = new Hashtable<>(); - for (IPropertyValueDecoder getter : getters) { - for (Class clazz : getter.getSupportedTypes()) { - /* - * We could validate that we never overwrite a getter but we can also do this by manual inspection of - * the getters. - */ - classToDecoder.put(clazz, getter); - } - } - nameToMethods = new Hashtable<>(); - for (Method method : KinesisClientLibConfiguration.class.getMethods()) { - if (!nameToMethods.containsKey(method.getName())) { - nameToMethods.put(method.getName(), new ArrayList<>()); - } - nameToMethods.get(method.getName()).add(method); - } - } - - /** - * Return a KinesisClientLibConfiguration with variables configured as specified by the properties in config stream. - * Program will fail immediately, if customer provide: 1) invalid variable value. Program will log it as warning and - * continue, if customer provide: 1) variable with unsupported variable type. 2) a variable with name which does not - * match any of the variables in KinesisClientLibConfigration. - * - * @param properties a Properties object containing the configuration information - * @return KinesisClientLibConfiguration - */ - public KinesisClientLibConfiguration getConfiguration(Properties properties) { - // The three minimum required arguments for constructor are obtained first. They are all mandatory, all of them - // should be provided. If any of these three failed to be set, program will fail. - IPropertyValueDecoder stringValueDecoder = new StringPropertyValueDecoder(); - IPropertyValueDecoder awsCPPropGetter = - new AWSCredentialsProviderPropertyValueDecoder(); - String applicationName = stringValueDecoder.decodeValue(properties.getProperty(PROP_APP_NAME)); - String streamName = stringValueDecoder.decodeValue(properties.getProperty(PROP_STREAM_NAME)); - AwsCredentialsProvider provider = - awsCPPropGetter.decodeValue(properties.getProperty(PROP_CREDENTIALS_PROVIDER_KINESIS)); - - if (applicationName == null || applicationName.isEmpty()) { - throw new IllegalArgumentException("Value of applicationName should be explicitly provided."); - } - if (streamName == null || streamName.isEmpty()) { - throw new IllegalArgumentException("Value of streamName should be explicitly provided."); - } - - // Decode the DynamoDB credentials provider if it exists. If not use the Kinesis credentials provider. - AwsCredentialsProvider providerDynamoDB; - String propCredentialsProviderDynamoDBValue = properties.getProperty(PROP_CREDENTIALS_PROVIDER_DYNAMODB); - if (propCredentialsProviderDynamoDBValue == null) { - providerDynamoDB = provider; - } else { - providerDynamoDB = awsCPPropGetter.decodeValue(propCredentialsProviderDynamoDBValue); - } - - // Decode the CloudWatch credentials provider if it exists. If not use the Kinesis credentials provider. - AwsCredentialsProvider providerCloudWatch; - String propCredentialsProviderCloudWatchValue = properties.getProperty(PROP_CREDENTIALS_PROVIDER_CLOUDWATCH); - if (propCredentialsProviderCloudWatchValue == null) { - providerCloudWatch = provider; - } else { - providerCloudWatch = awsCPPropGetter.decodeValue(propCredentialsProviderCloudWatchValue); - } - - // Allow customer not to provide workerId or to provide empty worker id. - String workerId = stringValueDecoder.decodeValue(properties.getProperty(PROP_WORKER_ID)); - if (workerId == null || workerId.isEmpty()) { - workerId = UUID.randomUUID().toString(); - log.info("Value of workerId is not provided in the properties. WorkerId is automatically assigned as: {}", - workerId); - } - - KinesisClientLibConfiguration config = - new KinesisClientLibConfiguration(applicationName, streamName, provider, providerDynamoDB, providerCloudWatch, workerId); - - Set requiredNames = - new HashSet(Arrays.asList(PROP_STREAM_NAME, - PROP_APP_NAME, - PROP_WORKER_ID, - PROP_CREDENTIALS_PROVIDER_KINESIS)); - - // Set all the variables that are not used for constructor. - for (Object keyObject : properties.keySet()) { - String key = keyObject.toString(); - if (!requiredNames.contains(key)) { - withProperty(key, properties, config); - } - } - - return config; - } - - /** - * @param configStream the input stream containing the configuration information - * @return KinesisClientLibConfiguration - */ - public KinesisClientLibConfiguration getConfiguration(InputStream configStream) { - Properties properties = new Properties(); - try { - properties.load(configStream); - } catch (IOException e) { - String msg = "Could not load properties from the stream provided"; - throw new IllegalStateException(msg, e); - } finally { - try { - configStream.close(); - } catch (IOException e) { - String msg = "Encountered error while trying to close properties file."; - throw new IllegalStateException(msg, e); - } - } - return getConfiguration(properties); - } - - private void withProperty(String propertyKey, Properties properties, KinesisClientLibConfiguration config) { - if (propertyKey.isEmpty()) { - throw new IllegalArgumentException("The property can't be empty string"); - } - // Assume that all the setters in KinesisClientLibConfiguration are in the following format - // They all start with "with" followed by the variable name with first letter capitalized - String targetMethodName = PREFIX + Character.toUpperCase(propertyKey.charAt(0)) + propertyKey.substring(1); - String propertyValue = properties.getProperty(propertyKey); - if (nameToMethods.containsKey(targetMethodName)) { - for (Method method : nameToMethods.get(targetMethodName)) { - if (method.getParameterTypes().length == 1 && method.getName().equals(targetMethodName)) { - Class paramType = method.getParameterTypes()[0]; - if (classToDecoder.containsKey(paramType)) { - IPropertyValueDecoder decoder = classToDecoder.get(paramType); - try { - method.invoke(config, decoder.decodeValue(propertyValue)); - log.info("Successfully set property {} with value {}", propertyKey, propertyValue); - return; - } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { - // At this point, we really thought that we could call this method. - log.warn("Encountered an error while invoking method %s with value {}. Exception was {}", - method, propertyValue, e); - } catch (UnsupportedOperationException e) { - log.warn("The property {} is not supported as type {} at this time.", propertyKey, - paramType); - } - } else { - log.debug("No method for decoding parameters of type {} so method {} could not be invoked.", - paramType, method); - } - } else { - log.debug("Method {} doesn't look like it is appropriate for setting property {}. Looking for" - + " something called {}.", method, propertyKey, targetMethodName); - } - } - } else { - log.debug(String.format("There was no appropriately named method for setting property %s.", propertyKey)); - } - } -} diff --git a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/LongPropertyValueDecoder.java b/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/LongPropertyValueDecoder.java deleted file mode 100644 index 1382b153..00000000 --- a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/LongPropertyValueDecoder.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/asl/ - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ -package com.amazonaws.services.kinesis.multilang.config; - -import java.util.Arrays; -import java.util.List; - -/** - * Get long type properties. - */ -class LongPropertyValueDecoder implements IPropertyValueDecoder { - - /** - * Constructor. - */ - LongPropertyValueDecoder() { - } - - /** - * @param value property value as String - * @return corresponding variable in correct type - */ - @Override - public Long decodeValue(String value) { - return Long.parseLong(value); - } - - /** - * @return list of supported types - */ - @Override - public List> getSupportedTypes() { - return Arrays.asList(long.class, Long.class); - } - -} diff --git a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/SetPropertyValueDecoder.java b/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/SetPropertyValueDecoder.java deleted file mode 100644 index 6dfe2dbe..00000000 --- a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/SetPropertyValueDecoder.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/asl/ - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ -package com.amazonaws.services.kinesis.multilang.config; - -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -/** - * Provide {@link Set} property value. Note that since parameterized value cannot be figured out during compile time - * for setter methods, only {@code Set} of {@code String}s are supported as property value decode. - */ -@SuppressWarnings("rawtypes") -class SetPropertyValueDecoder implements IPropertyValueDecoder { - - /** - * Delimiter for the list provided as string. - */ - private static final String LIST_DELIMITER = ","; - - /** - * Package constructor for factory use only. - */ - SetPropertyValueDecoder() { - } - - /** - * {@inheritDoc} - */ - @Override - public Set decodeValue(String propertyValue) { - String[] values = propertyValue.split(LIST_DELIMITER); - String value = null; - Set decodedValue = new HashSet<>(); - for (int i = 0; i < values.length; i++) { - value = values[i].trim(); - if (!value.isEmpty()) { - decodedValue.add(value); - } - } - return decodedValue; - } - - /** - * {@inheritDoc} - */ - @Override - public List> getSupportedTypes() { - return Arrays.asList(Set.class); - } - -} diff --git a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/StringPropertyValueDecoder.java b/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/StringPropertyValueDecoder.java deleted file mode 100644 index d5cc0482..00000000 --- a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/StringPropertyValueDecoder.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/asl/ - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ -package com.amazonaws.services.kinesis.multilang.config; - -import java.util.Arrays; -import java.util.List; - -/** - * Get string properties from properties file. - */ -class StringPropertyValueDecoder implements IPropertyValueDecoder { - - /** - * package constructor for factory use only. - */ - StringPropertyValueDecoder() { - } - - /** - * @param value the property value - * @return the value as String - */ - @Override - public String decodeValue(String value) { - // How to treat null or empty String should depend on those method who - // uses the String value. Here just return the string as it is. - if (value == null) { - return null; - } - return value.trim(); - } - - /** - * @return list of supported types - */ - @Override - public List> getSupportedTypes() { - return Arrays.asList(String.class); - } - -} diff --git a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/messages/StatusMessage.java b/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/messages/StatusMessage.java deleted file mode 100644 index 921cca1b..00000000 --- a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/messages/StatusMessage.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/asl/ - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ -package com.amazonaws.services.kinesis.multilang.messages; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.experimental.Accessors; - -/** - * A message sent by the client's process to indicate to the record processor that it completed a particular action. - */ -@NoArgsConstructor -@AllArgsConstructor -@Getter -@Setter -public class StatusMessage extends Message { - /** - * The name used for the action field in {@link Message}. - */ - public static final String ACTION = "status"; - - /** - * The name of the most recently received action. - */ - private String responseFor; -} diff --git a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/DrainChildSTDERRTask.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/DrainChildSTDERRTask.java similarity index 60% rename from amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/DrainChildSTDERRTask.java rename to amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/DrainChildSTDERRTask.java index 7276b229..d8236683 100644 --- a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/DrainChildSTDERRTask.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/DrainChildSTDERRTask.java @@ -1,18 +1,18 @@ /* - * Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package com.amazonaws.services.kinesis.multilang; +package software.amazon.kinesis.multilang; import java.io.BufferedReader; diff --git a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/DrainChildSTDOUTTask.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/DrainChildSTDOUTTask.java similarity index 97% rename from amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/DrainChildSTDOUTTask.java rename to amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/DrainChildSTDOUTTask.java index 0e95a14e..99f47c7f 100644 --- a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/DrainChildSTDOUTTask.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/DrainChildSTDOUTTask.java @@ -12,7 +12,7 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package com.amazonaws.services.kinesis.multilang; +package software.amazon.kinesis.multilang; import java.io.BufferedReader; diff --git a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/GetNextMessageTask.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/GetNextMessageTask.java similarity index 96% rename from amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/GetNextMessageTask.java rename to amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/GetNextMessageTask.java index 8177a8d2..15869a29 100644 --- a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/GetNextMessageTask.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/GetNextMessageTask.java @@ -12,12 +12,12 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package com.amazonaws.services.kinesis.multilang; +package software.amazon.kinesis.multilang; import java.io.BufferedReader; import java.io.IOException; -import com.amazonaws.services.kinesis.multilang.messages.Message; +import software.amazon.kinesis.multilang.messages.Message; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.extern.slf4j.Slf4j; diff --git a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/LineReaderTask.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/LineReaderTask.java similarity index 99% rename from amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/LineReaderTask.java rename to amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/LineReaderTask.java index 650fc0c5..6205ef53 100644 --- a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/LineReaderTask.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/LineReaderTask.java @@ -12,7 +12,7 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package com.amazonaws.services.kinesis.multilang; +package software.amazon.kinesis.multilang; import java.io.BufferedReader; import java.io.IOException; diff --git a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MessageReader.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MessageReader.java similarity index 87% rename from amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MessageReader.java rename to amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MessageReader.java index 6bd3aa93..60314c6f 100644 --- a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MessageReader.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MessageReader.java @@ -1,18 +1,18 @@ /* - * Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package com.amazonaws.services.kinesis.multilang; +package software.amazon.kinesis.multilang; import java.io.BufferedReader; import java.io.InputStream; @@ -20,7 +20,7 @@ import java.io.InputStreamReader; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; -import com.amazonaws.services.kinesis.multilang.messages.Message; +import software.amazon.kinesis.multilang.messages.Message; import com.fasterxml.jackson.databind.ObjectMapper; /** diff --git a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MessageWriter.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MessageWriter.java similarity index 82% rename from amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MessageWriter.java rename to amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MessageWriter.java index 164a36bf..5aea31fa 100644 --- a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MessageWriter.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MessageWriter.java @@ -12,7 +12,7 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package com.amazonaws.services.kinesis.multilang; +package software.amazon.kinesis.multilang; import java.io.BufferedWriter; import java.io.IOException; @@ -22,18 +22,20 @@ import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; -import com.amazonaws.services.kinesis.multilang.messages.CheckpointMessage; -import com.amazonaws.services.kinesis.multilang.messages.InitializeMessage; -import com.amazonaws.services.kinesis.multilang.messages.Message; -import com.amazonaws.services.kinesis.multilang.messages.ProcessRecordsMessage; -import com.amazonaws.services.kinesis.multilang.messages.ShutdownMessage; -import com.amazonaws.services.kinesis.multilang.messages.ShutdownRequestedMessage; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.extern.slf4j.Slf4j; import software.amazon.kinesis.lifecycle.events.InitializationInput; +import software.amazon.kinesis.lifecycle.events.LeaseLostInput; import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput; -import software.amazon.kinesis.lifecycle.ShutdownReason; +import software.amazon.kinesis.lifecycle.events.ShardEndedInput; +import software.amazon.kinesis.multilang.messages.CheckpointMessage; +import software.amazon.kinesis.multilang.messages.InitializeMessage; +import software.amazon.kinesis.multilang.messages.LeaseLostMessage; +import software.amazon.kinesis.multilang.messages.Message; +import software.amazon.kinesis.multilang.messages.ProcessRecordsMessage; +import software.amazon.kinesis.multilang.messages.ShardEndedMessage; +import software.amazon.kinesis.multilang.messages.ShutdownRequestedMessage; /** * Defines methods for writing {@link Message} objects to the child process's STDIN. @@ -135,12 +137,25 @@ class MessageWriter { } /** - * Writes a {@link ShutdownMessage} to the subprocess. + * Writes the lease lost message to the sub process. * - * @param reason The reason for shutting down. + * @param leaseLostInput + * the lease lost input. This is currently unused as lease loss doesn't actually have anything in it + * @return A future that is set when the message has been written. */ - Future writeShutdownMessage(ShutdownReason reason) { - return writeMessage(new ShutdownMessage(reason)); + Future writeLeaseLossMessage(@SuppressWarnings("unused") LeaseLostInput leaseLostInput) { + return writeMessage(new LeaseLostMessage()); + } + + /** + * Writes a message to the sub process indicating that the shard has ended + * + * @param shardEndedInput + * the shard end input. This is currently unused as the checkpoint is extracted, and used by the caller. + * @return A future that is set when the message has been written. + */ + Future writeShardEndedMessage(@SuppressWarnings("unused") ShardEndedInput shardEndedInput) { + return writeMessage(new ShardEndedMessage()); } /** diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MultiLangDaemon.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MultiLangDaemon.java new file mode 100644 index 00000000..79e8aed9 --- /dev/null +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MultiLangDaemon.java @@ -0,0 +1,234 @@ +/* + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package software.amazon.kinesis.multilang; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.LoggerFactory; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; + +import ch.qos.logback.classic.LoggerContext; +import ch.qos.logback.classic.joran.JoranConfigurator; +import ch.qos.logback.core.joran.spi.JoranException; +import lombok.Data; +import lombok.experimental.Accessors; +import lombok.extern.slf4j.Slf4j; +import software.amazon.kinesis.coordinator.Scheduler; + +/** + * Main app that launches the scheduler that runs the multi-language record processor. + * + * Requires a properties file containing configuration for this daemon and the KCL. A properties file should at minimum + * define these properties: + * + * 
+ * # The script that abides by the multi-language protocol. This script will
+ * # be executed by the MultiLangDaemon, which will communicate with this script
+ * # over STDIN and STDOUT according to the multi-language protocol.
+ * executableName = sampleapp.py
+ *
+ * # The name of an Amazon Kinesis stream to process.
+ * streamName = words
+ *
+ * # Used by the KCL as the name of this application. Will be used as the name
+ * # of a Amazon DynamoDB table which will store the lease and checkpoint
+ * # information for workers with this application name.
+ * applicationName = PythonKCLSample
+ *
+ * # Users can change the credentials provider the KCL will use to retrieve credentials.
+ * # The DefaultAWSCredentialsProviderChain checks several other providers, which is
+ * # described here:
+ * # http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html
+ * AWSCredentialsProvider = DefaultAWSCredentialsProviderChain
+ *
+ */ +@Slf4j +public class MultiLangDaemon { + static class MultiLangDaemonArguments { + @Parameter + List parameters = new ArrayList<>(); + + @Parameter(names = { "-p", "--properties-file" }, description = "Properties file to be used with the KCL") + String propertiesFile; + + @Parameter(names = { "-l", + "--log-configuration" }, description = "File location of logback.xml to be override the default") + String logConfiguration; + } + + @Data + @Accessors(fluent = true) + static class MultiLangRunner implements Callable { + private final Scheduler scheduler; + + @Override + public Integer call() throws Exception { + int exitCode = 0; + try { + scheduler().run(); + } catch (Throwable t) { + log.error("Caught throwable while processing data", t); + exitCode = 1; + } + return exitCode; + } + } + + JCommander buildJCommanderAndParseArgs(final MultiLangDaemonArguments arguments, final String[] args) { + JCommander jCommander = JCommander.newBuilder().programName("amazon-kinesis-client MultiLangDaemon") + .addObject(arguments) + .build(); + jCommander.parse(args); + return jCommander; + } + + void printUsage(final JCommander jCommander, final String message) { + if (StringUtils.isNotEmpty(message)) { + System.err.println(message); + } + jCommander.usage(); + } + + Scheduler buildScheduler(final MultiLangDaemonConfig config) { + return config.getMultiLangDaemonConfiguration().build(config.getRecordProcessorFactory()); + } + + void configureLogging(final String logConfiguration) { + if (StringUtils.isNotEmpty(logConfiguration)) { + LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory(); + JoranConfigurator configurator = new JoranConfigurator(); + configureLogging(logConfiguration, loggerContext, configurator); + } + } + + void configureLogging(final String logConfiguration, final LoggerContext loggerContext, + final JoranConfigurator configurator) { + loggerContext.reset(); + try (InputStream inputStream = FileUtils.openInputStream(new File(logConfiguration))) { + configurator.setContext(loggerContext); + configurator.doConfigure(inputStream); + } catch (IOException | JoranException e) { + throw new RuntimeException("Error while loading log configuration: " + e.getMessage()); + } + } + + String propertiesFile(final MultiLangDaemonArguments arguments) { + String propertiesFile = ""; + + if (CollectionUtils.isNotEmpty(arguments.parameters)) { + if (arguments.parameters.size() == 1) { + propertiesFile = arguments.parameters.get(0); + } else { + throw new RuntimeException( + "Expected a single argument, but found multiple arguments. Arguments: " + + String.join(", ", arguments.parameters)); + } + } + + if (StringUtils.isNotEmpty(arguments.propertiesFile)) { + if (StringUtils.isNotEmpty(propertiesFile)) { + log.warn("Overriding the properties file with the --properties-file option"); + } + propertiesFile = arguments.propertiesFile; + } + + if (StringUtils.isEmpty(propertiesFile)) { + throw new RuntimeException("Properties file missing, please provide a properties file"); + } + + return propertiesFile; + } + + MultiLangDaemonConfig buildMultiLangDaemonConfig(final String propertiesFile) { + try { + return new MultiLangDaemonConfig(propertiesFile); + } catch (IOException e) { + throw new RuntimeException("Error while reading properties file: " + e.getMessage()); + } + } + + void setupShutdownHook(final Runtime runtime, final MultiLangRunner runner, final MultiLangDaemonConfig config) { + long shutdownGraceMillis = config.getMultiLangDaemonConfiguration().getShutdownGraceMillis(); + runtime.addShutdownHook(new Thread(() -> { + log.info("Process terminated, will initiate shutdown."); + try { + Future runnerFuture = runner.scheduler().startGracefulShutdown(); + runnerFuture.get(shutdownGraceMillis, TimeUnit.MILLISECONDS); + log.info("Process shutdown is complete."); + } catch (InterruptedException | ExecutionException | TimeoutException e) { + log.error("Encountered an error during shutdown.", e); + } + })); + } + + int submitRunnerAndWait(final MultiLangDaemonConfig config, final MultiLangRunner runner) { + ExecutorService executorService = config.getExecutorService(); + Future future = executorService.submit(runner); + + try { + return future.get(); + } catch (InterruptedException | ExecutionException e) { + log.error("Encountered an error while running daemon", e); + } + return 1; + } + + void exit(final int exitCode) { + System.exit(exitCode); + } + + /** + * @param args + * Accepts a single argument, that argument is a properties file which provides KCL configuration as + * well as the name of an executable. + */ + public static void main(final String[] args) { + int exitCode = 1; + MultiLangDaemon daemon = new MultiLangDaemon(); + MultiLangDaemonArguments arguments = new MultiLangDaemonArguments(); + JCommander jCommander = daemon.buildJCommanderAndParseArgs(arguments, args); + try { + String propertiesFile = daemon.propertiesFile(arguments); + daemon.configureLogging(arguments.logConfiguration); + MultiLangDaemonConfig config = daemon.buildMultiLangDaemonConfig(propertiesFile); + + Scheduler scheduler = daemon.buildScheduler(config); + MultiLangRunner runner = new MultiLangRunner(scheduler); + + daemon.setupShutdownHook(Runtime.getRuntime(), runner, config); + exitCode = daemon.submitRunnerAndWait(config, runner); + } catch (Throwable t) { + t.printStackTrace(System.err); + daemon.printUsage(jCommander, t.getMessage()); + System.err.println("For more information, visit: https://github.com/awslabs/amazon-kinesis-client"); + } + daemon.exit(exitCode); + } +} diff --git a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangDaemonConfig.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MultiLangDaemonConfig.java similarity index 89% rename from amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangDaemonConfig.java rename to amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MultiLangDaemonConfig.java index 70f90a06..d8f37b34 100644 --- a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangDaemonConfig.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MultiLangDaemonConfig.java @@ -5,7 +5,7 @@ * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific * language governing permissions and limitations under the License. */ -package com.amazonaws.services.kinesis.multilang; +package software.amazon.kinesis.multilang; import java.io.File; import java.io.FileInputStream; @@ -19,11 +19,11 @@ import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import software.amazon.kinesis.coordinator.KinesisClientLibConfiguration; -import com.amazonaws.services.kinesis.multilang.config.KinesisClientLibConfigurator; +import software.amazon.kinesis.multilang.config.KinesisClientLibConfigurator; import com.google.common.util.concurrent.ThreadFactoryBuilder; import lombok.extern.slf4j.Slf4j; +import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration; import software.amazon.kinesis.retrieval.RetrievalConfig; /** @@ -38,7 +38,7 @@ public class MultiLangDaemonConfig { private static final String PROP_PROCESSING_LANGUAGE = "processingLanguage"; private static final String PROP_MAX_ACTIVE_THREADS = "maxActiveThreads"; - private KinesisClientLibConfiguration kinesisClientLibConfig; + private MultiLangDaemonConfiguration multiLangDaemonConfiguration; private ExecutorService executorService; @@ -98,13 +98,13 @@ public class MultiLangDaemonConfig { String executableName = properties.getProperty(PROP_EXECUTABLE_NAME); String processingLanguage = properties.getProperty(PROP_PROCESSING_LANGUAGE); - kinesisClientLibConfig = configurator.getConfiguration(properties); + multiLangDaemonConfiguration = configurator.getConfiguration(properties); executorService = buildExecutorService(properties); recordProcessorFactory = new MultiLangRecordProcessorFactory(executableName, executorService, - kinesisClientLibConfig); + multiLangDaemonConfiguration); - log.info("Running {} to process stream {} with executable {}", kinesisClientLibConfig.getApplicationName(), - kinesisClientLibConfig.getStreamName(), executableName); + log.info("Running {} to process stream {} with executable {}", multiLangDaemonConfiguration.getApplicationName(), + multiLangDaemonConfiguration.getStreamName(), executableName); prepare(processingLanguage); } @@ -112,9 +112,7 @@ public class MultiLangDaemonConfig { // Ensure the JVM will refresh the cached IP values of AWS resources (e.g. service endpoints). java.security.Security.setProperty("networkaddress.cache.ttl", "60"); - log.info("Using workerId: {}", kinesisClientLibConfig.getWorkerIdentifier()); - log.info("Using credentials with access key id: {}", - kinesisClientLibConfig.getKinesisCredentialsProvider().resolveCredentials().accessKeyId()); + log.info("Using workerId: {}", multiLangDaemonConfiguration.getWorkerIdentifier()); StringBuilder userAgent = new StringBuilder(RetrievalConfig.KINESIS_CLIENT_LIB_USER_AGENT); userAgent.append(" "); @@ -133,7 +131,7 @@ public class MultiLangDaemonConfig { } log.info("MultiLangDaemon is adding the following fields to the User Agent: {}", userAgent.toString()); - kinesisClientLibConfig.withUserAgent(userAgent.toString()); +// multiLangDaemonConfiguration.withUserAgent(userAgent.toString()); } private static Properties loadProperties(ClassLoader classLoader, String propertiesFileName) throws IOException { @@ -190,8 +188,8 @@ public class MultiLangDaemonConfig { * * @return A KinesisClientLibConfiguration object based on the properties file provided. */ - public KinesisClientLibConfiguration getKinesisClientLibConfiguration() { - return kinesisClientLibConfig; + public MultiLangDaemonConfiguration getMultiLangDaemonConfiguration() { + return multiLangDaemonConfiguration; } /** diff --git a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangProtocol.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MultiLangProtocol.java similarity index 82% rename from amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangProtocol.java rename to amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MultiLangProtocol.java index 75e552ce..7e9823e0 100644 --- a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangProtocol.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MultiLangProtocol.java @@ -12,7 +12,7 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package com.amazonaws.services.kinesis.multilang; +package software.amazon.kinesis.multilang; import java.util.Optional; import java.util.concurrent.ExecutionException; @@ -20,21 +20,22 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import software.amazon.kinesis.exceptions.InvalidStateException; -import software.amazon.kinesis.processor.RecordProcessorCheckpointer; -import software.amazon.kinesis.coordinator.KinesisClientLibConfiguration; -import software.amazon.kinesis.lifecycle.ShutdownReason; -import software.amazon.kinesis.lifecycle.events.InitializationInput; -import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput; -import com.amazonaws.services.kinesis.multilang.messages.CheckpointMessage; -import com.amazonaws.services.kinesis.multilang.messages.InitializeMessage; -import com.amazonaws.services.kinesis.multilang.messages.Message; -import com.amazonaws.services.kinesis.multilang.messages.ProcessRecordsMessage; -import com.amazonaws.services.kinesis.multilang.messages.ShutdownMessage; -import com.amazonaws.services.kinesis.multilang.messages.ShutdownRequestedMessage; -import com.amazonaws.services.kinesis.multilang.messages.StatusMessage; - import lombok.extern.slf4j.Slf4j; +import software.amazon.kinesis.exceptions.InvalidStateException; +import software.amazon.kinesis.lifecycle.events.InitializationInput; +import software.amazon.kinesis.lifecycle.events.LeaseLostInput; +import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput; +import software.amazon.kinesis.lifecycle.events.ShardEndedInput; +import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration; +import software.amazon.kinesis.multilang.messages.CheckpointMessage; +import software.amazon.kinesis.multilang.messages.InitializeMessage; +import software.amazon.kinesis.multilang.messages.LeaseLostMessage; +import software.amazon.kinesis.multilang.messages.Message; +import software.amazon.kinesis.multilang.messages.ProcessRecordsMessage; +import software.amazon.kinesis.multilang.messages.ShardEndedMessage; +import software.amazon.kinesis.multilang.messages.ShutdownRequestedMessage; +import software.amazon.kinesis.multilang.messages.StatusMessage; +import software.amazon.kinesis.processor.RecordProcessorCheckpointer; /** * An implementation of the multi language protocol. @@ -42,10 +43,13 @@ import lombok.extern.slf4j.Slf4j; @Slf4j class MultiLangProtocol { + private final InitializationInput initializationInput; + private final Optional timeoutInSeconds; + private MessageReader messageReader; private MessageWriter messageWriter; - private final InitializationInput initializationInput; - private KinesisClientLibConfiguration configuration; + + private MultiLangDaemonConfiguration configuration; /** * Constructor. @@ -58,11 +62,12 @@ class MultiLangProtocol { * information about the shard this processor is starting to process */ MultiLangProtocol(MessageReader messageReader, MessageWriter messageWriter, - InitializationInput initializationInput, KinesisClientLibConfiguration configuration) { + InitializationInput initializationInput, MultiLangDaemonConfiguration configuration) { this.messageReader = messageReader; this.messageWriter = messageWriter; this.initializationInput = initializationInput; this.configuration = configuration; + this.timeoutInSeconds = Optional.ofNullable(configuration.getTimeoutInSeconds()); } /** @@ -94,16 +99,24 @@ class MultiLangProtocol { } /** - * Writes a {@link ShutdownMessage} to the child process's STDIN and waits for the child process to respond with a - * {@link StatusMessage} on its STDOUT. - * - * @param checkpointer A checkpointer. - * @param reason Why this processor is being shutdown. - * @return Whether or not this operation succeeded. + * Notifies the client process that the lease has been lost, and it needs to shutdown. + * + * @param leaseLostInput + * the lease lost input that is passed to the {@link MessageWriter} + * @return true if the message was successfully writtem */ - boolean shutdown(RecordProcessorCheckpointer checkpointer, ShutdownReason reason) { - Future writeFuture = messageWriter.writeShutdownMessage(reason); - return waitForStatusMessage(ShutdownMessage.ACTION, checkpointer, writeFuture); + boolean leaseLost(LeaseLostInput leaseLostInput) { + return waitForStatusMessage(LeaseLostMessage.ACTION, null, messageWriter.writeLeaseLossMessage(leaseLostInput)); + } + + /** + * + * @param shardEndedInput + * @return + */ + boolean shardEnded(ShardEndedInput shardEndedInput) { + return waitForStatusMessage(ShardEndedMessage.ACTION, shardEndedInput.checkpointer(), + messageWriter.writeShardEndedMessage(shardEndedInput)); } /** @@ -164,18 +177,18 @@ class MultiLangProtocol { Optional statusMessage = Optional.empty(); while (!statusMessage.isPresent()) { Future future = this.messageReader.getNextMessageFromSTDOUT(); - Optional message = configuration.getTimeoutInSeconds() - .map(second -> futureMethod(() -> future.get(second, TimeUnit.SECONDS), action)) - .orElse(futureMethod(future::get, action)); + Optional message = timeoutInSeconds + .map(second -> futureMethod(() -> future.get(second, TimeUnit.SECONDS), action)) + .orElse(futureMethod(future::get, action)); if (!message.isPresent()) { return false; } - Optional checkpointFailed = message.filter(m -> m instanceof CheckpointMessage ) - .map(m -> (CheckpointMessage) m) - .flatMap(m -> futureMethod(() -> checkpoint(m, checkpointer).get(), "Checkpoint")) - .map(checkpointSuccess -> !checkpointSuccess); + Optional checkpointFailed = message.filter(m -> m instanceof CheckpointMessage) + .map(m -> (CheckpointMessage) m) + .flatMap(m -> futureMethod(() -> checkpoint(m, checkpointer).get(), "Checkpoint")) + .map(checkpointSuccess -> !checkpointSuccess); if (checkpointFailed.orElse(false)) { return false; diff --git a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangRecordProcessorFactory.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MultiLangRecordProcessorFactory.java similarity index 89% rename from amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangRecordProcessorFactory.java rename to amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MultiLangRecordProcessorFactory.java index 734e6364..b1858a63 100644 --- a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangRecordProcessorFactory.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MultiLangRecordProcessorFactory.java @@ -12,7 +12,7 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package com.amazonaws.services.kinesis.multilang; +package software.amazon.kinesis.multilang; import java.util.concurrent.ExecutorService; @@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import lombok.extern.slf4j.Slf4j; import software.amazon.kinesis.coordinator.KinesisClientLibConfiguration; +import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration; import software.amazon.kinesis.processor.ShardRecordProcessorFactory; import software.amazon.kinesis.processor.ShardRecordProcessor; @@ -37,14 +38,14 @@ public class MultiLangRecordProcessorFactory implements ShardRecordProcessorFact private final ExecutorService executorService; - private final KinesisClientLibConfiguration configuration; + private final MultiLangDaemonConfiguration configuration; /** * @param command The command that will do processing for this factory's record processors. * @param executorService An executor service to use while processing inputs and outputs of the child process. */ public MultiLangRecordProcessorFactory(String command, ExecutorService executorService, - KinesisClientLibConfiguration configuration) { + MultiLangDaemonConfiguration configuration) { this(command, executorService, new ObjectMapper(), configuration); } @@ -54,7 +55,7 @@ public class MultiLangRecordProcessorFactory implements ShardRecordProcessorFact * @param objectMapper An object mapper used to convert messages to json to be written to the child process */ public MultiLangRecordProcessorFactory(String command, ExecutorService executorService, ObjectMapper objectMapper, - KinesisClientLibConfiguration configuration) { + MultiLangDaemonConfiguration configuration) { this.command = command; this.commandArray = command.split(COMMAND_DELIMETER_REGEX); this.executorService = executorService; diff --git a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangShardRecordProcessor.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MultiLangShardRecordProcessor.java similarity index 93% rename from amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangShardRecordProcessor.java rename to amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MultiLangShardRecordProcessor.java index 94df3c36..ae7996db 100644 --- a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/MultiLangShardRecordProcessor.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MultiLangShardRecordProcessor.java @@ -12,26 +12,25 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package com.amazonaws.services.kinesis.multilang; +package software.amazon.kinesis.multilang; import java.io.IOException; import java.io.InputStream; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import java.util.function.Function; -import software.amazon.kinesis.lifecycle.ShutdownReason; -import software.amazon.kinesis.lifecycle.events.LeaseLostInput; -import software.amazon.kinesis.lifecycle.events.ShardEndedInput; -import software.amazon.kinesis.lifecycle.events.ShutdownRequestedInput; -import software.amazon.kinesis.processor.ShardRecordProcessor; -import software.amazon.kinesis.coordinator.KinesisClientLibConfiguration; -import software.amazon.kinesis.lifecycle.events.InitializationInput; -import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput; -import software.amazon.kinesis.lifecycle.ShutdownInput; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.extern.slf4j.Slf4j; +import software.amazon.kinesis.lifecycle.events.InitializationInput; +import software.amazon.kinesis.lifecycle.events.LeaseLostInput; +import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput; +import software.amazon.kinesis.lifecycle.events.ShardEndedInput; +import software.amazon.kinesis.lifecycle.events.ShutdownRequestedInput; +import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration; +import software.amazon.kinesis.processor.ShardRecordProcessor; /** @@ -64,7 +63,7 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor { private MultiLangProtocol protocol; - private KinesisClientLibConfiguration configuration; + private MultiLangDaemonConfiguration configuration; @Override public void initialize(InitializationInput initializationInput) { @@ -113,12 +112,12 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor { @Override public void leaseLost(LeaseLostInput leaseLostInput) { - shutdown(ShutdownInput.builder().shutdownReason(ShutdownReason.LEASE_LOST).build()); + shutdown(p -> p.leaseLost(leaseLostInput)); } @Override public void shardEnded(ShardEndedInput shardEndedInput) { - shutdown(ShutdownInput.builder().shutdownReason(ShutdownReason.SHARD_END).checkpointer(shardEndedInput.checkpointer()).build()); + shutdown(p -> p.shardEnded(shardEndedInput)); } @Override @@ -134,7 +133,7 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor { } } - void shutdown(ShutdownInput shutdownInput) { + void shutdown(Function protocolInvocation) { // In cases where KCL loses lease for the shard after creating record processor instance but before // record processor initialize() is called, then shutdown() may be called directly before initialize(). if (!initialized) { @@ -146,7 +145,7 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor { try { if (ProcessState.ACTIVE.equals(this.state)) { - if (!protocol.shutdown(shutdownInput.checkpointer(), shutdownInput.shutdownReason())) { + if (!protocolInvocation.apply(protocol)) { throw new RuntimeException("Child process failed to shutdown"); } @@ -182,7 +181,7 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor { * An obejct mapper. */ MultiLangShardRecordProcessor(ProcessBuilder processBuilder, ExecutorService executorService, - ObjectMapper objectMapper, KinesisClientLibConfiguration configuration) { + ObjectMapper objectMapper, MultiLangDaemonConfiguration configuration) { this(processBuilder, executorService, objectMapper, new MessageWriter(), new MessageReader(), new DrainChildSTDERRTask(), configuration); } @@ -205,7 +204,7 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor { */ MultiLangShardRecordProcessor(ProcessBuilder processBuilder, ExecutorService executorService, ObjectMapper objectMapper, MessageWriter messageWriter, MessageReader messageReader, DrainChildSTDERRTask readSTDERRTask, - KinesisClientLibConfiguration configuration) { + MultiLangDaemonConfiguration configuration) { this.executorService = executorService; this.processBuilder = processBuilder; this.objectMapper = objectMapper; diff --git a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoder.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoder.java similarity index 83% rename from amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoder.java rename to amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoder.java index f6e1883c..103741ca 100644 --- a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoder.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoder.java @@ -12,13 +12,15 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package com.amazonaws.services.kinesis.multilang.config; +package software.amazon.kinesis.multilang.config; import java.lang.reflect.Constructor; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.AWSCredentialsProviderChain; import lombok.extern.slf4j.Slf4j; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain; @@ -27,7 +29,7 @@ import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain; * Get AWSCredentialsProvider property. */ @Slf4j -class AWSCredentialsProviderPropertyValueDecoder implements IPropertyValueDecoder { +class AWSCredentialsProviderPropertyValueDecoder implements IPropertyValueDecoder { private static final String AUTH_PREFIX = "com.amazonaws.auth."; private static final String LIST_DELIMITER = ","; private static final String ARG_DELIMITER = "|"; @@ -46,13 +48,13 @@ class AWSCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode * @return corresponding variable in correct type */ @Override - public AwsCredentialsProvider decodeValue(String value) { + public AWSCredentialsProvider decodeValue(String value) { if (value != null) { List providerNames = getProviderNames(value); - List providers = getValidCredentialsProviders(providerNames); - AwsCredentialsProvider[] ps = new AwsCredentialsProvider[providers.size()]; + List providers = getValidCredentialsProviders(providerNames); + AWSCredentialsProvider[] ps = new AWSCredentialsProvider[providers.size()]; providers.toArray(ps); - return AwsCredentialsProviderChain.builder().credentialsProviders(ps).build(); + return new AWSCredentialsProviderChain(providers); } else { throw new IllegalArgumentException("Property AWSCredentialsProvider is missing."); } @@ -62,15 +64,15 @@ class AWSCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode * @return list of supported types */ @Override - public List> getSupportedTypes() { - return Arrays.asList(AwsCredentialsProvider.class); + public List> getSupportedTypes() { + return Arrays.asList(AWSCredentialsProvider.class); } /* * Convert string list to a list of valid credentials providers. */ - private static List getValidCredentialsProviders(List providerNames) { - List credentialsProviders = new ArrayList(); + private static List getValidCredentialsProviders(List providerNames) { + List credentialsProviders = new ArrayList<>(); for (String providerName : providerNames) { if (providerName.contains(ARG_DELIMITER)) { String[] nameAndArgs = providerName.split("\\" + ARG_DELIMITER); @@ -79,7 +81,7 @@ class AWSCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode try { Class className = Class.forName(nameAndArgs[0]); Constructor c = className.getConstructor(argTypes); - credentialsProviders.add((AwsCredentialsProvider) c + credentialsProviders.add((AWSCredentialsProvider) c .newInstance(Arrays.copyOfRange(nameAndArgs, 1, nameAndArgs.length))); } catch (Exception e) { log.debug("Can't find any credentials provider matching {}.", providerName); @@ -87,7 +89,7 @@ class AWSCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode } else { try { Class className = Class.forName(providerName); - credentialsProviders.add((AwsCredentialsProvider) className.newInstance()); + credentialsProviders.add((AWSCredentialsProvider) className.newInstance()); } catch (Exception e) { log.debug("Can't find any credentials provider matching {}.", providerName); } diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/BuilderDynaBean.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/BuilderDynaBean.java new file mode 100644 index 00000000..1a4d01fe --- /dev/null +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/BuilderDynaBean.java @@ -0,0 +1,286 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.kinesis.multilang.config; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.function.Function; +import java.util.function.Supplier; + +import lombok.Getter; +import org.apache.commons.beanutils.ConvertUtilsBean; +import org.apache.commons.beanutils.DynaBean; +import org.apache.commons.beanutils.DynaClass; +import org.apache.commons.beanutils.DynaProperty; +import org.apache.commons.lang3.ClassUtils; +import org.apache.commons.lang3.StringUtils; + +public class BuilderDynaBean implements DynaBean { + + private static final String[] CLASS_NAME_JOINERS = { ClassUtils.PACKAGE_SEPARATOR, ClassUtils.INNER_CLASS_SEPARATOR }; + static final String NO_MAP_ACCESS_SUPPORT = "Map access isn't supported"; + + private Class destinedClass; + private final ConvertUtilsBean convertUtilsBean; + private final List classPrefixSearchList; + + private DynaBeanCreateSupport dynaBeanCreateSupport; + private DynaBeanBuilderSupport dynaBeanBuilderSupport; + + @Getter + private boolean isDirty = false; + + private final Function emptyPropertyHandler; + private Object emptyPropertyResolved = null; + + public BuilderDynaBean(Class destinedClass, ConvertUtilsBean convertUtilsBean, String... classPrefixSearchList) { + this(destinedClass, convertUtilsBean, null, Arrays.asList(classPrefixSearchList)); + } + + public BuilderDynaBean(Class destinedClass, ConvertUtilsBean convertUtilsBean, + Function emptyPropertyHandler, String... classPrefixSearchList) { + this(destinedClass, convertUtilsBean, emptyPropertyHandler, Arrays.asList(classPrefixSearchList)); + } + + public BuilderDynaBean(Class destinedClass, ConvertUtilsBean convertUtilsBean, + Function emtpyPropertyHandler, List classPrefixSearchList) { + this.convertUtilsBean = convertUtilsBean; + this.classPrefixSearchList = classPrefixSearchList; + this.emptyPropertyHandler = emtpyPropertyHandler; + initialize(destinedClass); + } + + private void initialize(Class destinedClass) { + this.destinedClass = destinedClass; + + if (DynaBeanBuilderUtils.isBuilderOrCreate(destinedClass)) { + dynaBeanBuilderSupport = new DynaBeanBuilderSupport(destinedClass, convertUtilsBean, classPrefixSearchList); + dynaBeanCreateSupport = new DynaBeanCreateSupport(destinedClass, convertUtilsBean, classPrefixSearchList); + } + } + + private void reinitializeFrom(String newClass) { + Class newClazz = null; + List attempts = new ArrayList<>(); + attempts.add(newClass); + try { + newClazz = Class.forName(newClass); + } catch (ClassNotFoundException e) { + // + // Ignored + // + } + if (newClazz == null) { + for (String prefix : classPrefixSearchList) { + for (String joiner : CLASS_NAME_JOINERS) { + String possibleClass; + if (prefix.endsWith(joiner)) { + possibleClass = prefix + newClass; + } else { + possibleClass = prefix + joiner + newClass; + } + attempts.add(possibleClass); + try { + newClazz = Class.forName(possibleClass); + break; + } catch (ClassNotFoundException e) { + // + // Ignored + // + } + + } + } + } + + if (newClazz == null) { + throw new IllegalArgumentException( + "Unable to load class " + newClass + ". Attempted: (" + String.join(", ", attempts) + ")"); + } + initialize(newClazz); + } + + private void validatedExpectedClass(Class source, Class expected) { + if (!ClassUtils.isAssignable(source, expected)) { + throw new IllegalArgumentException( + String.format("%s cannot be assigned to %s.", source.getName(), expected.getName())); + } + } + + public boolean canBuildOrCreate() { + return dynaBeanBuilderSupport != null || dynaBeanCreateSupport != null; + } + + private void validateCanBuildOrCreate() { + if (!canBuildOrCreate()) { + throw new IllegalStateException("Unable to to introspect or handle " + destinedClass.getName() + + " as it doesn't have a builder or create method."); + } + } + + @SafeVarargs + public final T build(Class expected, Function... additionalMutators) { + if (emptyPropertyResolved != null) { + validatedExpectedClass(emptyPropertyResolved.getClass(), expected); + return expected.cast(emptyPropertyResolved); + } + + if (dynaBeanBuilderSupport == null && dynaBeanCreateSupport == null) { + return null; + } + + validatedExpectedClass(destinedClass, expected); + if (dynaBeanBuilderSupport.isValid()) { + return expected.cast(dynaBeanBuilderSupport.build(additionalMutators)); + } else { + return expected.cast(dynaBeanCreateSupport.build()); + } + + } + + private void validateResolvedEmptyHandler() { + if (emptyPropertyResolved != null) { + throw new IllegalStateException("When a property handler is resolved further properties may not be set."); + } + } + + boolean hasValue(String name) { + if (dynaBeanBuilderSupport != null) { + return dynaBeanBuilderSupport.hasValue(name); + } + return false; + } + + @Override + public boolean contains(String name, String key) { + throw new UnsupportedOperationException(NO_MAP_ACCESS_SUPPORT); + } + + @Override + public Object get(String name) { + validateResolvedEmptyHandler(); + isDirty = true; + return dynaBeanBuilderSupport.get(name); + } + + @Override + public Object get(String name, int index) { + validateResolvedEmptyHandler(); + isDirty = true; + if (StringUtils.isEmpty(name)) { + return dynaBeanCreateSupport.get(name, index); + } + return dynaBeanBuilderSupport.get(name, index); + } + + @Override + public Object get(String name, String key) { + throw new UnsupportedOperationException(NO_MAP_ACCESS_SUPPORT); + } + + @Override + public DynaClass getDynaClass() { + return new DynaClass() { + @Override + public String getName() { + return destinedClass.getName(); + } + + @Override + public DynaProperty getDynaProperty(String name) { + if (StringUtils.isEmpty(name)) { + return new DynaProperty(name); + } + if ("class".equals(name)) { + return new DynaProperty(name, String.class); + } + // + // We delay validation until after the class check to allow for re-initialization for a specific class. + // The check for isEmpty is allowed ahead of this check to allow for raw string support. + // + validateCanBuildOrCreate(); + List types = dynaBeanBuilderSupport.getProperty(name); + if (types.size() > 1) { + Optional arrayType = types.stream().filter(t -> t.type.isArray()).findFirst(); + return arrayType.map(t -> new DynaProperty(name, t.type, t.type.getComponentType())) + .orElseGet(() -> new DynaProperty(name)); + } else { + TypeTag type = types.get(0); + if (type.hasConverter) { + return new DynaProperty(name, type.type); + } + if (type.type.isEnum()) { + return new DynaProperty(name, String.class); + } + return new DynaProperty(name, BuilderDynaBean.class); + } + } + + @Override + public DynaProperty[] getDynaProperties() { + validateCanBuildOrCreate(); + return dynaBeanBuilderSupport.getPropertyNames().stream().map(this::getDynaProperty) + .toArray(DynaProperty[]::new); + } + + @Override + public DynaBean newInstance() { + return null; + } + }; + } + + @Override + public void remove(String name, String key) { + throw new UnsupportedOperationException(NO_MAP_ACCESS_SUPPORT); + } + + @Override + public void set(String name, Object value) { + validateResolvedEmptyHandler(); + isDirty = true; + if (emptyPropertyHandler != null && StringUtils.isEmpty(name) && value instanceof String) { + emptyPropertyResolved = emptyPropertyHandler.apply((String) value); + return; + } + + if ("class".equals(name)) { + reinitializeFrom(value.toString()); + } else { + validateResolvedEmptyHandler(); + dynaBeanBuilderSupport.set(name, value); + } + } + + @Override + public void set(String name, int index, Object value) { + validateResolvedEmptyHandler(); + validateCanBuildOrCreate(); + isDirty = true; + if (StringUtils.isEmpty(name)) { + dynaBeanCreateSupport.set(name, index, value); + } else { + dynaBeanBuilderSupport.set(name, index, value); + } + } + + @Override + public void set(String name, String key, Object value) { + throw new UnsupportedOperationException(NO_MAP_ACCESS_SUPPORT); + } +} diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/ConfigurationSettable.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/ConfigurationSettable.java new file mode 100644 index 00000000..3dbb4e6b --- /dev/null +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/ConfigurationSettable.java @@ -0,0 +1,49 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.kinesis.multilang.config; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Repeatable; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.FIELD) +@Repeatable(ConfigurationSettables.class) +public @interface ConfigurationSettable { + + /** + * Which builder this option applies to + * + * @return the class of the builder to use + */ + Class configurationClass(); + + /** + * The method name on the builder, defaults to the fieldName + * + * @return the name of the method or null to use the default + */ + String methodName() default ""; + + /** + * If the type is actually an optional value this will enable conversions + * + * @return true if the value should be wrapped by an optional + */ + boolean convertToOptional() default false; +} diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/ConfigurationSettableUtils.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/ConfigurationSettableUtils.java new file mode 100644 index 00000000..28e01cd4 --- /dev/null +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/ConfigurationSettableUtils.java @@ -0,0 +1,109 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.kinesis.multilang.config; + +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import org.apache.commons.lang3.ClassUtils; +import org.apache.commons.lang3.StringUtils; + +import com.google.common.base.Defaults; + +import lombok.NonNull; + +public class ConfigurationSettableUtils { + + public static T resolveFields(@NonNull Object source, @NonNull T configObject) { + Map, Object> configObjects = new HashMap<>(); + configObjects.put(configObject.getClass(), configObject); + resolveFields(source, configObjects, null, null); + + return configObject; + } + + public static void resolveFields(Object source, Map, Object> configObjects, Set> restrictTo, + Set> skipIf) { + for (Field field : source.getClass().getDeclaredFields()) { + for (ConfigurationSettable b : field.getAnnotationsByType(ConfigurationSettable.class)) { + if (restrictTo != null && !restrictTo.contains(b.configurationClass())) { + continue; + } + if (skipIf != null && skipIf.contains(b.configurationClass())) { + continue; + } + field.setAccessible(true); + Object configObject = configObjects.get(b.configurationClass()); + if (configObject != null) { + String setterName = field.getName(); + if (!StringUtils.isEmpty(b.methodName())) { + setterName = b.methodName(); + } + Object value; + try { + value = field.get(source); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + + if (value != null && !value.equals(Defaults.defaultValue(field.getType()))) { + Method setter = null; + if (b.convertToOptional()) { + value = Optional.of(value); + } + if (ClassUtils.isPrimitiveOrWrapper(value.getClass())) { + Class primitiveType = field.getType().isPrimitive() ? field.getType() + : ClassUtils.wrapperToPrimitive(field.getType()); + Class wrapperType = !field.getType().isPrimitive() ? field.getType() + : ClassUtils.primitiveToWrapper(field.getType()); + + try { + setter = b.configurationClass().getMethod(setterName, primitiveType); + } catch (NoSuchMethodException e) { + // + // Ignore this + // + } + if (setter == null) { + try { + setter = b.configurationClass().getMethod(setterName, wrapperType); + } catch (NoSuchMethodException e) { + throw new RuntimeException(e); + } + } + } else { + try { + setter = b.configurationClass().getMethod(setterName, value.getClass()); + } catch (NoSuchMethodException e) { + throw new RuntimeException(e); + } + } + try { + setter.invoke(configObject, value); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException(e); + } + } + } + } + } + } +} diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/ConfigurationSettables.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/ConfigurationSettables.java new file mode 100644 index 00000000..1fadc7c9 --- /dev/null +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/ConfigurationSettables.java @@ -0,0 +1,27 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.kinesis.multilang.config; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.FIELD) +public @interface ConfigurationSettables { + ConfigurationSettable[] value(); +} diff --git a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/DatePropertyValueDecoder.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/DatePropertyValueDecoder.java similarity index 96% rename from amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/DatePropertyValueDecoder.java rename to amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/DatePropertyValueDecoder.java index 591c90cc..4344047b 100644 --- a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/DatePropertyValueDecoder.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/DatePropertyValueDecoder.java @@ -12,7 +12,7 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package com.amazonaws.services.kinesis.multilang.config; +package software.amazon.kinesis.multilang.config; import java.util.Arrays; import java.util.Date; diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/DynaBeanBuilderSupport.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/DynaBeanBuilderSupport.java new file mode 100644 index 00000000..e3d5cb7d --- /dev/null +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/DynaBeanBuilderSupport.java @@ -0,0 +1,253 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.kinesis.multilang.config; + +import java.lang.reflect.Array; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; + +import org.apache.commons.beanutils.ConvertUtilsBean; +import org.apache.commons.lang3.ClassUtils; + +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimap; + +class DynaBeanBuilderSupport { + + private static final String BUILD_METHOD_NAME = "build"; + private static final String BUILDER_METHOD_NAME = "builder"; + + private final Class destinedClass; + private final ConvertUtilsBean convertUtilsBean; + private final List classPrefixSearchList; + private final Class builderClass; + + private final Multimap properties = HashMultimap.create(); + private final Map values = new HashMap<>(); + + DynaBeanBuilderSupport(Class destinedClass, ConvertUtilsBean convertUtilsBean, + List classPrefixSearchList) { + this.destinedClass = destinedClass; + this.convertUtilsBean = convertUtilsBean; + this.classPrefixSearchList = classPrefixSearchList; + this.builderClass = builderClassFrom(destinedClass); + + buildProperties(); + } + + private static Class builderClassFrom(Class destinedClass) { + Method builderMethod; + try { + builderMethod = destinedClass.getMethod(BUILDER_METHOD_NAME); + } catch (NoSuchMethodException e) { + return null; + } + + return builderMethod.getReturnType(); + } + + private void buildProperties() { + if (builderClass == null) { + return; + } + try { + builderClass.getMethod(BUILD_METHOD_NAME); + } catch (NoSuchMethodException e) { + throw new RuntimeException(e); + } + + for (Method method : builderClass.getMethods()) { + if (method.getParameterCount() == 1 && ClassUtils.isAssignable(builderClass, method.getReturnType())) { + Class paramType = method.getParameterTypes()[0]; + if (Supplier.class.isAssignableFrom(paramType) || Consumer.class.isAssignableFrom(paramType)) { + continue; + } + if (paramType.isEnum()) { + properties.put(method.getName(), new TypeTag(paramType, true, method)); + } else if (convertUtilsBean.lookup(paramType) == null) { + properties.put(method.getName(), new TypeTag(paramType, false, method)); + } else { + properties.put(method.getName(), new TypeTag(paramType, true, method)); + } + } + } + } + + boolean isValid() { + return builderClass != null; + } + + private Object createForProperty(String name) { + Optional type = properties.get(name).stream().findFirst(); + return type.map(t -> { + if (DynaBeanBuilderUtils.isBuilderOrCreate(t.type) || !t.hasConverter) { + return new BuilderDynaBean(t.type, convertUtilsBean, null, classPrefixSearchList); + } + return null; + }).orElse(null); + } + + boolean hasValue(String name) { + return values.containsKey(name); + } + + Object get(String name) { + if (values.containsKey(name)) { + return values.get(name); + } + Object value = createForProperty(name); + if (value != null) { + values.put(name, value); + } + return values.get(name); + } + + private Object[] retrieveAndResizeArray(String name, int index) { + Object existing = values.get(name); + Object[] destination; + if (existing != null) { + if (!existing.getClass().isArray()) { + throw new IllegalStateException("Requested get for an array, but existing value isn't an array"); + } + destination = (Object[]) existing; + if (index >= destination.length) { + destination = Arrays.copyOf(destination, index + 1); + values.put(name, destination); + } + + } else { + destination = new Object[index + 1]; + values.put(name, destination); + } + + return destination; + } + + Object get(String name, int index) { + Object[] destination = retrieveAndResizeArray(name, index); + + if (destination[index] == null) { + destination[index] = createForProperty(name); + } + return destination[index]; + } + + void set(String name, Object value) { + if (value instanceof String && properties.get(name).stream().anyMatch(t -> t.type.isEnum())) { + TypeTag typeTag = properties.get(name).stream().filter(t -> t.type.isEnum()).findFirst().orElseThrow( + () -> new IllegalStateException("Expected enum type for " + name + ", but couldn't find it.")); + Class enumClass = (Class) typeTag.type; + values.put(name, Enum.valueOf(enumClass, value.toString())); + } else { + values.put(name, value); + } + } + + void set(String name, int index, Object value) { + Object[] destination = retrieveAndResizeArray(name, index); + destination[index] = value; + } + + private Object getArgument(Map.Entry setValue) { + Object argument = setValue.getValue(); + if (argument instanceof Object[]) { + TypeTag arrayType = properties.get(setValue.getKey()).stream().filter(t -> t.type.isArray()).findFirst() + .orElseThrow(() -> new IllegalStateException(String + .format("Received Object[] for %s but can't find corresponding type", setValue.getKey()))); + Object[] arrayValues = (Object[]) argument; + Object[] destination = (Object[]) Array.newInstance(arrayType.type.getComponentType(), arrayValues.length); + + for (int i = 0; i < arrayValues.length; ++i) { + if (arrayValues[i] instanceof BuilderDynaBean) { + destination[i] = ((BuilderDynaBean) arrayValues[i]).build(Object.class); + } else { + destination[i] = arrayValues[i]; + } + } + + return destination; + } + if (argument instanceof BuilderDynaBean) { + argument = ((BuilderDynaBean) argument).build(Object.class); + } + return argument; + } + + Object build(Function... additionalMutators) { + Method builderMethod; + try { + builderMethod = destinedClass.getMethod(BUILDER_METHOD_NAME); + } catch (NoSuchMethodException e) { + throw new RuntimeException(e); + } + Object source; + try { + source = builderMethod.invoke(null); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException(e); + } + for (Map.Entry setValue : values.entrySet()) { + Object argument = getArgument(setValue); + Method mutator = properties.get(setValue.getKey()).stream() + .filter(t -> ClassUtils.isAssignable(argument.getClass(), t.type)).findFirst() + .map(a -> a.builderMethod).orElseThrow( + () -> new IllegalStateException(String.format("Unable to find mutator for %s of type %s", + setValue.getKey(), argument.getClass().getName()))); + try { + source = mutator.invoke(source, argument); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException(e); + } + } + + if (additionalMutators != null) { + for (Function mutator : additionalMutators) { + source = mutator.apply(source); + } + } + + Method buildMethod; + try { + buildMethod = builderClass.getMethod(BUILD_METHOD_NAME); + return buildMethod.invoke(source); + } catch (IllegalAccessException | NoSuchMethodException | InvocationTargetException e) { + throw new RuntimeException(e); + } + + } + + Collection getPropertyNames() { + return properties.keySet(); + } + + List getProperty(String name) { + if (!properties.containsKey(name)) { + throw new IllegalArgumentException("Unknown property: " + name); + } + return new ArrayList<>(properties.get(name)); + } + +} diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/DynaBeanBuilderUtils.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/DynaBeanBuilderUtils.java new file mode 100644 index 00000000..f5d85d6d --- /dev/null +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/DynaBeanBuilderUtils.java @@ -0,0 +1,56 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.kinesis.multilang.config; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Arrays; + +public class DynaBeanBuilderUtils { + + static Method getMethod(Class clazz, String name, Class... parameterTypes) { + try { + return clazz.getMethod(name, parameterTypes); + } catch (NoSuchMethodException e) { + throw new RuntimeException(e); + } + } + + static Object invokeOrFail(Method method, Object onObject, Object... arguments) { + try { + return method.invoke(onObject, arguments); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException(e); + } + } + + static boolean isBuilderOrCreate(Class clazz) { + Method buildMethod = null; + + try { + buildMethod = clazz.getMethod("builder"); + } catch (NoSuchMethodException e) { + // + // Ignored + // + } + + boolean hasCreate = Arrays.stream(clazz.getMethods()) + .anyMatch(m -> "create".equals(m.getName()) && m.getReturnType().isAssignableFrom(clazz)); + + return buildMethod != null || hasCreate; + } +} diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/DynaBeanCreateSupport.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/DynaBeanCreateSupport.java new file mode 100644 index 00000000..5aab9511 --- /dev/null +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/DynaBeanCreateSupport.java @@ -0,0 +1,101 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.kinesis.multilang.config; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.beanutils.ConvertUtilsBean; +import org.apache.commons.lang3.StringUtils; + +class DynaBeanCreateSupport { + + private final Class destinedClass; + private final ConvertUtilsBean convertUtilsBean; + private final List classPrefixSearchList; + private final List createTypes = new ArrayList<>(); + private Object[] createValues = null; + + DynaBeanCreateSupport(Class destinedClass, ConvertUtilsBean convertUtilsBean, + List classPrefixSearchList) { + this.destinedClass = destinedClass; + this.convertUtilsBean = convertUtilsBean; + this.classPrefixSearchList = classPrefixSearchList; + + readTypes(); + } + + private void readTypes() { + for (Method method : destinedClass.getMethods()) { + if ("create".equals(method.getName()) && method.getReturnType().isAssignableFrom(destinedClass)) { + createValues = new Object[method.getParameterCount()]; + int i = 0; + for (Class paramType : method.getParameterTypes()) { + if (convertUtilsBean.lookup(paramType) != null) { + createTypes.add(new TypeTag(paramType, true, null)); + } else { + createTypes.add(new TypeTag(paramType, false, null)); + } + ++i; + } + } + } + } + + Object build() { + + Method createMethod = DynaBeanBuilderUtils.getMethod(destinedClass, "create", + createTypes.stream().map(t -> t.type).toArray(i -> new Class[i])); + Object arguments[] = new Object[createValues.length]; + for (int i = 0; i < createValues.length; ++i) { + if (createValues[i] instanceof BuilderDynaBean) { + arguments[i] = ((BuilderDynaBean) createValues[i]).build(createTypes.get(i).type); + } else { + arguments[i] = createValues[i]; + } + } + return DynaBeanBuilderUtils.invokeOrFail(createMethod, null, arguments); + } + + public Object get(String name, int index) { + if (index < createValues.length) { + if (createTypes.get(index).hasConverter) { + return createValues[index]; + } else { + if (createValues[index] == null) { + createValues[index] = new BuilderDynaBean(createTypes.get(index).type, convertUtilsBean, null, + classPrefixSearchList); + } + return createValues[index]; + } + } + return null; + } + + public void set(String name, int index, Object value) { + if (StringUtils.isEmpty(name)) { + if (index >= createValues.length) { + throw new IllegalArgumentException( + String.format("%d exceeds the maximum number of arguments (%d) for %s", index, + createValues.length, destinedClass.getName())); + } + createValues[index] = value; + } + + } + +} diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/FanoutConfigBean.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/FanoutConfigBean.java new file mode 100644 index 00000000..5576142e --- /dev/null +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/FanoutConfigBean.java @@ -0,0 +1,46 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.kinesis.multilang.config; + +import lombok.Getter; +import lombok.Setter; +import software.amazon.awssdk.services.kinesis.KinesisAsyncClient; +import software.amazon.kinesis.retrieval.fanout.FanOutConfig; + +@Getter +@Setter +public class FanoutConfigBean implements RetrievalConfigBuilder { + + @ConfigurationSettable(configurationClass = FanOutConfig.class) + private int maxDescribeStreamSummaryRetries; + @ConfigurationSettable(configurationClass = FanOutConfig.class) + private String consumerArn; + @ConfigurationSettable(configurationClass = FanOutConfig.class) + private String consumerName; + @ConfigurationSettable(configurationClass = FanOutConfig.class) + private int maxDescribeStreamConsumerRetries; + @ConfigurationSettable(configurationClass = FanOutConfig.class) + private int registerStreamConsumerRetries; + @ConfigurationSettable(configurationClass = FanOutConfig.class) + private long retryBackoffMillis; + + @Override + public FanOutConfig build(KinesisAsyncClient kinesisAsyncClient, MultiLangDaemonConfiguration parent) { + return ConfigurationSettableUtils.resolveFields(this, new FanOutConfig(kinesisAsyncClient).applicationName(parent.getApplicationName()) + .streamName(parent.getStreamName())); + } + +} diff --git a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/IPropertyValueDecoder.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/IPropertyValueDecoder.java similarity index 95% rename from amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/IPropertyValueDecoder.java rename to amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/IPropertyValueDecoder.java index bc23b18b..6bc22b09 100644 --- a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/IPropertyValueDecoder.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/IPropertyValueDecoder.java @@ -12,7 +12,7 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package com.amazonaws.services.kinesis.multilang.config; +package software.amazon.kinesis.multilang.config; import java.util.List; diff --git a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/IntegerPropertyValueDecoder.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/IntegerPropertyValueDecoder.java similarity index 95% rename from amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/IntegerPropertyValueDecoder.java rename to amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/IntegerPropertyValueDecoder.java index 012ea2b6..969be3cb 100644 --- a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/config/IntegerPropertyValueDecoder.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/IntegerPropertyValueDecoder.java @@ -12,7 +12,7 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package com.amazonaws.services.kinesis.multilang.config; +package software.amazon.kinesis.multilang.config; import java.util.Arrays; import java.util.List; diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/KinesisClientLibConfigurator.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/KinesisClientLibConfigurator.java new file mode 100644 index 00000000..f0907060 --- /dev/null +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/KinesisClientLibConfigurator.java @@ -0,0 +1,100 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package software.amazon.kinesis.multilang.config; + +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.InvocationTargetException; +import java.util.Properties; + +import org.apache.commons.beanutils.BeanUtilsBean; +import org.apache.commons.beanutils.ConvertUtilsBean; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Validate; + +/** + * KinesisClientLibConfigurator constructs a KinesisClientLibConfiguration from java properties file. The following + * three properties must be provided. 1) "applicationName" 2) "streamName" 3) "AWSCredentialsProvider" + * KinesisClientLibConfigurator will help to automatically assign the value of "workerId" if this property is not + * provided. In the specified properties file, any properties, which matches the variable name in + * KinesisClientLibConfiguration and has a corresponding "with{variableName}" setter method, will be read in, and its + * value in properties file will be assigned to corresponding variable in KinesisClientLibConfiguration. + */ +@Slf4j +public class KinesisClientLibConfigurator { + private final ConvertUtilsBean convertUtilsBean; + private final BeanUtilsBean utilsBean; + private final MultiLangDaemonConfiguration configuration; + + + /** + * Constructor. + */ + public KinesisClientLibConfigurator() { + this.convertUtilsBean = new ConvertUtilsBean(); + this.utilsBean = new BeanUtilsBean(convertUtilsBean); + this.configuration = new MultiLangDaemonConfiguration(utilsBean, convertUtilsBean); + } + + /** + * Return a KinesisClientLibConfiguration with variables configured as specified by the properties in config stream. + * Program will fail immediately, if customer provide: 1) invalid variable value. Program will log it as warning and + * continue, if customer provide: 1) variable with unsupported variable type. 2) a variable with name which does not + * match any of the variables in KinesisClientLibConfigration. + * + * @param properties a Properties object containing the configuration information + * @return KinesisClientLibConfiguration + */ + public MultiLangDaemonConfiguration getConfiguration(Properties properties) { + properties.entrySet().forEach(e -> { + try { + utilsBean.setProperty(configuration, (String) e.getKey(), e.getValue()); + } catch (IllegalAccessException | InvocationTargetException ex) { + throw new RuntimeException(ex); + } + }); + + Validate.notBlank(configuration.getApplicationName(), "Application name is required"); + Validate.notBlank(configuration.getStreamName(), "Stream name is required"); + Validate.isTrue(configuration.getKinesisCredentialsProvider().isDirty(), "A basic set of AWS credentials must be provided"); + return configuration; + } + + /** + * @param configStream the input stream containing the configuration information + * @return KinesisClientLibConfiguration + */ + public MultiLangDaemonConfiguration getConfiguration(InputStream configStream) { + Properties properties = new Properties(); + try { + properties.load(configStream); + } catch (IOException e) { + String msg = "Could not load properties from the stream provided"; + throw new IllegalStateException(msg, e); + } finally { + try { + configStream.close(); + } catch (IOException e) { + String msg = "Encountered error while trying to close properties file."; + throw new IllegalStateException(msg, e); + } + } + return getConfiguration(properties); + } + + +} diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/MultiLangDaemonConfiguration.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/MultiLangDaemonConfiguration.java new file mode 100644 index 00000000..bb1d3dfa --- /dev/null +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/MultiLangDaemonConfiguration.java @@ -0,0 +1,394 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.kinesis.multilang.config; + +import java.lang.reflect.InvocationTargetException; +import java.net.URI; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; + +import org.apache.commons.beanutils.BeanUtilsBean; +import org.apache.commons.beanutils.ConvertUtilsBean; +import org.apache.commons.beanutils.Converter; +import org.apache.commons.beanutils.converters.ArrayConverter; +import org.apache.commons.beanutils.converters.StringConverter; + +import lombok.Data; +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.Delegate; +import lombok.extern.slf4j.Slf4j; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient; +import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient; +import software.amazon.awssdk.services.kinesis.KinesisAsyncClient; +import software.amazon.awssdk.services.kinesis.KinesisAsyncClientBuilder; +import software.amazon.kinesis.checkpoint.CheckpointConfig; +import software.amazon.kinesis.common.ConfigsBuilder; +import software.amazon.kinesis.common.InitialPositionInStream; +import software.amazon.kinesis.common.InitialPositionInStreamExtended; +import software.amazon.kinesis.common.KinesisClientUtil; +import software.amazon.kinesis.coordinator.CoordinatorConfig; +import software.amazon.kinesis.coordinator.Scheduler; +import software.amazon.kinesis.leases.LeaseManagementConfig; +import software.amazon.kinesis.leases.ShardPrioritization; +import software.amazon.kinesis.lifecycle.LifecycleConfig; +import software.amazon.kinesis.metrics.MetricsConfig; +import software.amazon.kinesis.metrics.MetricsLevel; +import software.amazon.kinesis.multilang.config.credentials.V2CredentialWrapper; +import software.amazon.kinesis.processor.ProcessorConfig; +import software.amazon.kinesis.processor.ShardRecordProcessorFactory; +import software.amazon.kinesis.retrieval.RetrievalConfig; +import software.amazon.kinesis.retrieval.polling.PollingConfig; + +@Getter +@Setter +@Slf4j +public class MultiLangDaemonConfiguration { + + private static final String CREDENTIALS_DEFAULT_SEARCH_PATH = "software.amazon.awssdk.auth.credentials"; + + private String applicationName; + + private String streamName; + + @ConfigurationSettable(configurationClass = ConfigsBuilder.class) + private String tableName; + + private String workerIdentifier = UUID.randomUUID().toString(); + + public void setWorkerId(String workerId) { + this.workerIdentifier = workerId; + } + + @ConfigurationSettable(configurationClass = LeaseManagementConfig.class) + private long failoverTimeMillis; + @ConfigurationSettable(configurationClass = LeaseManagementConfig.class) + private long shardSyncIntervalMillis; + @ConfigurationSettable(configurationClass = LeaseManagementConfig.class) + private boolean cleanupLeasesUponShardCompletion; + @ConfigurationSettable(configurationClass = LeaseManagementConfig.class) + private boolean ignoreUnexpectedChildShards; + @ConfigurationSettable(configurationClass = LeaseManagementConfig.class) + private int maxLeasesForWorker; + @ConfigurationSettable(configurationClass = LeaseManagementConfig.class) + private int maxLeasesToStealAtOneTime; + @ConfigurationSettable(configurationClass = LeaseManagementConfig.class) + private int initialLeaseTableReadCapacity; + @ConfigurationSettable(configurationClass = LeaseManagementConfig.class) + private int initialLeaseTableWriteCapacity; + @ConfigurationSettable(configurationClass = LeaseManagementConfig.class, methodName = "initialPositionInStream") + @ConfigurationSettable(configurationClass = RetrievalConfig.class) + private InitialPositionInStreamExtended initialPositionInStreamExtended; + + public InitialPositionInStream getInitialPositionInStream() { + if (initialPositionInStreamExtended != null) { + return initialPositionInStreamExtended.getInitialPositionInStream(); + } + return null; + } + + public void setInitialPositionInStream(InitialPositionInStream initialPositionInStream) { + this.initialPositionInStreamExtended = InitialPositionInStreamExtended + .newInitialPosition(initialPositionInStream); + } + + @ConfigurationSettable(configurationClass = LeaseManagementConfig.class) + private int maxLeaseRenewalThreads; + @ConfigurationSettable(configurationClass = LeaseManagementConfig.class) + private long listShardsBackoffTimeInMillis; + @ConfigurationSettable(configurationClass = LeaseManagementConfig.class) + private int maxListShardsRetryAttempts; + + // Enables applications flush/checkpoint (if they have some data "in progress", but don't get new data for while) + @ConfigurationSettable(configurationClass = ProcessorConfig.class) + private boolean callProcessRecordsEvenForEmptyRecordList; + + @ConfigurationSettable(configurationClass = CoordinatorConfig.class) + private long parentShardPollIntervalMillis; + @ConfigurationSettable(configurationClass = CoordinatorConfig.class) + private ShardPrioritization shardPrioritization; + @ConfigurationSettable(configurationClass = CoordinatorConfig.class) + private boolean skipShardSyncAtWorkerInitializationIfLeasesExist; + + @ConfigurationSettable(configurationClass = LifecycleConfig.class) + private long taskBackoffTimeMillis; + + @ConfigurationSettable(configurationClass = MetricsConfig.class) + private long metricsBufferTimeMillis; + @ConfigurationSettable(configurationClass = MetricsConfig.class) + private int metricsMaxQueueSize; + @ConfigurationSettable(configurationClass = MetricsConfig.class) + private MetricsLevel metricsLevel; + @ConfigurationSettable(configurationClass = LifecycleConfig.class, convertToOptional = true) + private Long logWarningForTaskAfterMillis; + @ConfigurationSettable(configurationClass = MetricsConfig.class) + private Set metricsEnabledDimensions; + + public String[] getMetricsEnabledDimensions() { + return metricsEnabledDimensions.toArray(new String[0]); + } + + public void setMetricsEnabledDimensions(String[] dimensions) { + metricsEnabledDimensions = new HashSet<>(Arrays.asList(dimensions)); + } + + + private RetrievalMode retrievalMode = RetrievalMode.DEFAULT; + + private final FanoutConfigBean fanoutConfig = new FanoutConfigBean(); + @Delegate(types = PollingConfigBean.PollingConfigBeanDelegate.class) + private final PollingConfigBean pollingConfig = new PollingConfigBean(); + + private boolean validateSequenceNumberBeforeCheckpointing; + + private long shutdownGraceMillis; + private Integer timeoutInSeconds; + + + private final BuilderDynaBean kinesisCredentialsProvider; + + public void setAWSCredentialsProvider(String providerString) { + kinesisCredentialsProvider.set("", providerString); + } + + private final BuilderDynaBean dynamoDBCredentialsProvider; + + public void setAWSCredentialsProviderDynamoDB(String providerString) { + dynamoDBCredentialsProvider.set("", providerString); + } + + private final BuilderDynaBean cloudWatchCredentialsProvider; + + public void setAWSCredentialsProviderCloudWatch(String providerString) { + cloudWatchCredentialsProvider.set("", providerString); + } + + private final BuilderDynaBean kinesisClient; + private final BuilderDynaBean dynamoDbClient; + private final BuilderDynaBean cloudWatchClient; + + private final BeanUtilsBean utilsBean; + private final ConvertUtilsBean convertUtilsBean; + + public MultiLangDaemonConfiguration(BeanUtilsBean utilsBean, ConvertUtilsBean convertUtilsBean) { + this.utilsBean = utilsBean; + this.convertUtilsBean = convertUtilsBean; + + convertUtilsBean.register(new Converter() { + @Override + public T convert(Class type, Object value) { + return type.cast(MetricsLevel.valueOf(value.toString().toUpperCase())); + } + }, MetricsLevel.class); + + convertUtilsBean.register(new Converter() { + @Override + public T convert(Class type, Object value) { + return type.cast(InitialPositionInStream.valueOf(value.toString().toUpperCase())); + } + }, InitialPositionInStream.class); + + convertUtilsBean.register(new Converter() { + @Override + public T convert(Class type, Object value) { + return type.cast(URI.create(value.toString())); + } + }, URI.class); + + convertUtilsBean.register(new Converter() { + @Override + public T convert(Class type, Object value) { + return type.cast(RetrievalMode.from(value.toString())); + } + }, RetrievalMode.class); + + convertUtilsBean.register(new Converter() { + @Override + public T convert(final Class type, final Object value) { + return type.cast(Region.of(value.toString())); + } + }, Region.class); + + ArrayConverter arrayConverter = new ArrayConverter(String[].class, new StringConverter()); + arrayConverter.setDelimiter(','); + convertUtilsBean.register(arrayConverter, String[].class); + AWSCredentialsProviderPropertyValueDecoder oldCredentialsDecoder = new AWSCredentialsProviderPropertyValueDecoder(); + Function converter = s -> new V2CredentialWrapper(oldCredentialsDecoder.decodeValue(s)); + + this.kinesisCredentialsProvider = new BuilderDynaBean(AwsCredentialsProvider.class, convertUtilsBean, + converter, CREDENTIALS_DEFAULT_SEARCH_PATH); + this.dynamoDBCredentialsProvider = new BuilderDynaBean(AwsCredentialsProvider.class, convertUtilsBean, + converter, CREDENTIALS_DEFAULT_SEARCH_PATH); + this.cloudWatchCredentialsProvider = new BuilderDynaBean(AwsCredentialsProvider.class, convertUtilsBean, + converter, CREDENTIALS_DEFAULT_SEARCH_PATH); + + this.kinesisClient = new BuilderDynaBean(KinesisAsyncClient.class, convertUtilsBean); + this.dynamoDbClient = new BuilderDynaBean(DynamoDbAsyncClient.class, convertUtilsBean); + this.cloudWatchClient = new BuilderDynaBean(CloudWatchAsyncClient.class, convertUtilsBean); + } + + private void setRegionForClient(String name, BuilderDynaBean client, Region region) { + try { + utilsBean.setProperty(client, "region", region); + } catch (IllegalAccessException | InvocationTargetException e) { + log.error("Failed to set region on {}", name, e); + throw new IllegalStateException(e); + } + } + + public void setRegionName(Region region) { + setRegionForClient("kinesisClient", kinesisClient, region); + setRegionForClient("dynamoDbClient", dynamoDbClient, region); + setRegionForClient("cloudWatchClient", cloudWatchClient, region); + } + + private void setEndpointForClient(String name, BuilderDynaBean client, String endpoint) { + try { + utilsBean.setProperty(client, "endpointOverride", endpoint); + } catch (IllegalAccessException | InvocationTargetException e) { + log.error("Failed to set endpoint on {}", name, e); + throw new IllegalStateException(e); + } + } + + public void setKinesisEndpoint(String endpoint) { + setEndpointForClient("kinesisClient", kinesisClient, endpoint); + } + + public void setDynamoDBEndpoint(String endpoint) { + setEndpointForClient("dynamoDbClient", dynamoDbClient, endpoint); + } + + private AwsCredentialsProvider resolveCredentials(BuilderDynaBean credsBuilder) { + if (!credsBuilder.isDirty()) { + return null; + } + return credsBuilder.build(AwsCredentialsProvider.class); + } + + private void updateCredentials(BuilderDynaBean toUpdate, AwsCredentialsProvider primary, + AwsCredentialsProvider secondary) { + + if (toUpdate.hasValue("credentialsProvider")) { + return; + } + + try { + if (primary != null) { + utilsBean.setProperty(toUpdate, "credentialsProvider", primary); + } else if (secondary != null) { + utilsBean.setProperty(toUpdate, "credentialsProvider", secondary); + } + } catch (IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException("Unable to update credentials", e); + } + } + + private void addConfigObjects(Map, Object> configObjects, Object... toAdd) { + for (Object obj : toAdd) { + configObjects.put(obj.getClass(), obj); + } + } + + private void resolveFields(Map, Object> configObjects, Set> restrictTo, Set> skipIf) { + ConfigurationSettableUtils.resolveFields(this, configObjects, restrictTo, skipIf); + } + + private void handleRetrievalConfig(RetrievalConfig retrievalConfig, ConfigsBuilder configsBuilder) { + retrievalConfig + .retrievalSpecificConfig(retrievalMode.builder(this).build(configsBuilder.kinesisClient(), this)); + } + + private Object adjustKinesisHttpConfiguration(Object builderObj) { + if (builderObj instanceof KinesisAsyncClientBuilder) { + KinesisAsyncClientBuilder builder = (KinesisAsyncClientBuilder) builderObj; + return builder.applyMutation(KinesisClientUtil::adjustKinesisClientBuilder); + } + + return builderObj; + } + + @Data + static class ResolvedConfiguration { + final CoordinatorConfig coordinatorConfig; + final CheckpointConfig checkpointConfig; + final LeaseManagementConfig leaseManagementConfig; + final LifecycleConfig lifecycleConfig; + final MetricsConfig metricsConfig; + final ProcessorConfig processorConfig; + final RetrievalConfig retrievalConfig; + + public Scheduler build() { + return new Scheduler(checkpointConfig, coordinatorConfig, leaseManagementConfig, lifecycleConfig, + metricsConfig, processorConfig, retrievalConfig); + } + } + + ResolvedConfiguration resolvedConfiguration(ShardRecordProcessorFactory shardRecordProcessorFactory) { + AwsCredentialsProvider kinesisCreds = resolveCredentials(kinesisCredentialsProvider); + AwsCredentialsProvider dynamoDbCreds = resolveCredentials(dynamoDBCredentialsProvider); + AwsCredentialsProvider cloudwatchCreds = resolveCredentials(cloudWatchCredentialsProvider); + + updateCredentials(kinesisClient, kinesisCreds, kinesisCreds); + updateCredentials(dynamoDbClient, dynamoDbCreds, kinesisCreds); + updateCredentials(cloudWatchClient, cloudwatchCreds, kinesisCreds); + + KinesisAsyncClient kinesisAsyncClient = kinesisClient.build(KinesisAsyncClient.class, + this::adjustKinesisHttpConfiguration); + DynamoDbAsyncClient dynamoDbAsyncClient = dynamoDbClient.build(DynamoDbAsyncClient.class); + CloudWatchAsyncClient cloudWatchAsyncClient = cloudWatchClient.build(CloudWatchAsyncClient.class); + + ConfigsBuilder configsBuilder = new ConfigsBuilder(streamName, applicationName, kinesisAsyncClient, + dynamoDbAsyncClient, cloudWatchAsyncClient, workerIdentifier, shardRecordProcessorFactory); + + Map, Object> configObjects = new HashMap<>(); + addConfigObjects(configObjects, configsBuilder); + + resolveFields(configObjects, Collections.singleton(ConfigsBuilder.class), + Collections.singleton(PollingConfig.class)); + + CoordinatorConfig coordinatorConfig = configsBuilder.coordinatorConfig(); + CheckpointConfig checkpointConfig = configsBuilder.checkpointConfig(); + LeaseManagementConfig leaseManagementConfig = configsBuilder.leaseManagementConfig(); + LifecycleConfig lifecycleConfig = configsBuilder.lifecycleConfig(); + MetricsConfig metricsConfig = configsBuilder.metricsConfig(); + ProcessorConfig processorConfig = configsBuilder.processorConfig(); + RetrievalConfig retrievalConfig = configsBuilder.retrievalConfig(); + + addConfigObjects(configObjects, coordinatorConfig, checkpointConfig, leaseManagementConfig, lifecycleConfig, + metricsConfig, processorConfig, retrievalConfig); + + handleRetrievalConfig(retrievalConfig, configsBuilder); + + resolveFields(configObjects, null, new HashSet<>(Arrays.asList(ConfigsBuilder.class, PollingConfig.class))); + + return new ResolvedConfiguration(coordinatorConfig, checkpointConfig, leaseManagementConfig, lifecycleConfig, + metricsConfig, processorConfig, retrievalConfig); + } + + public Scheduler build(ShardRecordProcessorFactory shardRecordProcessorFactory) { + return resolvedConfiguration(shardRecordProcessorFactory).build(); + } + +} diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/PollingConfigBean.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/PollingConfigBean.java new file mode 100644 index 00000000..6a362c44 --- /dev/null +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/PollingConfigBean.java @@ -0,0 +1,63 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.kinesis.multilang.config; + +import lombok.Getter; +import lombok.Setter; +import software.amazon.awssdk.services.kinesis.KinesisAsyncClient; +import software.amazon.kinesis.retrieval.polling.PollingConfig; + +@Getter +@Setter +public class PollingConfigBean implements RetrievalConfigBuilder { + + /** + * This is used to auto-generate a delegate by Lombok at {@link MultiLangDaemonConfiguration#getPollingConfig()} + */ + interface PollingConfigBeanDelegate { + + Integer getRetryGetRecordsInSeconds(); + void setRetryGetRecordsInSeconds(Integer value); + + Integer getMaxGetRecordsThreadPool(); + void setMaxGetRecordsThreadPool(Integer value); + + long getIdleTimeBetweenReadsInMillis(); + void setIdleTimeBetweenReadsInMillis(long value); + + int getMaxRecords(); + void setMaxRecords(int value); + } + + @ConfigurationSettable(configurationClass = PollingConfig.class, convertToOptional = true) + private Integer retryGetRecordsInSeconds; + @ConfigurationSettable(configurationClass = PollingConfig.class, convertToOptional = true) + private Integer maxGetRecordsThreadPool; + @ConfigurationSettable(configurationClass = PollingConfig.class) + private long idleTimeBetweenReadsInMillis; + @ConfigurationSettable(configurationClass = PollingConfig.class) + private int maxRecords; + + public boolean anyPropertiesSet() { + return retryGetRecordsInSeconds != null || maxGetRecordsThreadPool != null || idleTimeBetweenReadsInMillis != 0 || maxRecords != 0; + } + + @Override + public PollingConfig build(KinesisAsyncClient kinesisAsyncClient, MultiLangDaemonConfiguration parent) { + return ConfigurationSettableUtils.resolveFields(this, new PollingConfig(parent.getStreamName(), kinesisAsyncClient)); + } + +} diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/RetrievalConfigBuilder.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/RetrievalConfigBuilder.java new file mode 100644 index 00000000..6e3f023f --- /dev/null +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/RetrievalConfigBuilder.java @@ -0,0 +1,32 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.kinesis.multilang.config; + +import software.amazon.awssdk.services.kinesis.KinesisAsyncClient; +import software.amazon.kinesis.retrieval.RetrievalSpecificConfig; + +public interface RetrievalConfigBuilder { + /** + * Creates a retrieval specific configuration using the supplied parameters, and internal class parameters + * + * @param kinesisAsyncClient + * the client that will be provided to the RetrievalSpecificConfig constructor + * @param parent + * configuration parameters that this builder can access to configure it self + * @return a RetrievalSpecificConfig configured according to the customer's configuration. + */ + public RetrievalSpecificConfig build(KinesisAsyncClient kinesisAsyncClient, MultiLangDaemonConfiguration parent); +} diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/RetrievalMode.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/RetrievalMode.java new file mode 100644 index 00000000..b0ad5cbf --- /dev/null +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/RetrievalMode.java @@ -0,0 +1,64 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.kinesis.multilang.config; + +import java.util.Arrays; +import java.util.function.Function; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.Validate; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public enum RetrievalMode { + FANOUT(MultiLangDaemonConfiguration::getFanoutConfig), POLLING( + MultiLangDaemonConfiguration::getPollingConfig), DEFAULT(RetrievalMode::decideForDefault); + + private final Function builderFor; + + public RetrievalConfigBuilder builder(MultiLangDaemonConfiguration configuration) { + return builderFor.apply(configuration); + } + + RetrievalMode(Function builderFor) { + this.builderFor = builderFor; + } + + public static RetrievalMode from(String source) { + Validate.notEmpty(source); + try { + return RetrievalMode.valueOf(source.toUpperCase()); + } catch (IllegalArgumentException iae) { + throw new IllegalArgumentException( + "Unknown retrieval type '" + source + "'. Available retrieval types: " + availableRetrievalModes()); + } + } + + private static String availableRetrievalModes() { + return "(" + Arrays.stream(RetrievalMode.values()).map(Enum::name).collect(Collectors.joining(", ")) + ")"; + } + + private static RetrievalConfigBuilder decideForDefault(MultiLangDaemonConfiguration configuration) { + if (configuration.getPollingConfig().anyPropertiesSet()) { + log.warn("Some polling properties have been set, defaulting to polling. " + + "To switch to Fanout either add `RetrievalMode=FANOUT` to your " + + "properties or remove the any configuration for polling."); + return configuration.getPollingConfig(); + } + return configuration.getFanoutConfig(); + } +} diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/TypeTag.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/TypeTag.java new file mode 100644 index 00000000..0fde59c9 --- /dev/null +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/TypeTag.java @@ -0,0 +1,28 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.kinesis.multilang.config; + +import lombok.Data; + +import java.lang.reflect.Method; + +@Data +class TypeTag { + final Class type; + final boolean hasConverter; + final Method builderMethod; + +} diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/credentials/V2CredentialWrapper.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/credentials/V2CredentialWrapper.java new file mode 100644 index 00000000..b6445b3b --- /dev/null +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/credentials/V2CredentialWrapper.java @@ -0,0 +1,49 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.kinesis.multilang.config.credentials; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.AWSSessionCredentials; +import lombok.RequiredArgsConstructor; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; + +@RequiredArgsConstructor +public class V2CredentialWrapper implements AwsCredentialsProvider { + + private final AWSCredentialsProvider oldCredentialsProvider; + + @Override + public AwsCredentials resolveCredentials() { + AWSCredentials current = oldCredentialsProvider.getCredentials(); + if (current instanceof AWSSessionCredentials) { + return AwsSessionCredentials.create(current.getAWSSecretKey(), current.getAWSAccessKeyId(), ((AWSSessionCredentials) current).getSessionToken()); + } + return new AwsCredentials() { + @Override + public String accessKeyId() { + return current.getAWSAccessKeyId(); + } + + @Override + public String secretAccessKey() { + return current.getAWSSecretKey(); + } + }; + } +} diff --git a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/messages/CheckpointMessage.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/CheckpointMessage.java similarity index 95% rename from amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/messages/CheckpointMessage.java rename to amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/CheckpointMessage.java index 51159fc6..48de0b9b 100644 --- a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/messages/CheckpointMessage.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/CheckpointMessage.java @@ -12,12 +12,11 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package com.amazonaws.services.kinesis.multilang.messages; +package software.amazon.kinesis.multilang.messages; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; -import lombok.experimental.Accessors; /** * A checkpoint message is sent by the client's subprocess to indicate to the kcl processor that it should attempt to diff --git a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/messages/InitializeMessage.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/InitializeMessage.java similarity index 97% rename from amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/messages/InitializeMessage.java rename to amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/InitializeMessage.java index 4774e59a..c1279e75 100644 --- a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/messages/InitializeMessage.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/InitializeMessage.java @@ -12,7 +12,7 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package com.amazonaws.services.kinesis.multilang.messages; +package software.amazon.kinesis.multilang.messages; import lombok.Getter; import lombok.Setter; diff --git a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/messages/JsonFriendlyRecord.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/JsonFriendlyRecord.java similarity index 73% rename from amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/messages/JsonFriendlyRecord.java rename to amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/JsonFriendlyRecord.java index 5d4b0031..4e68f84f 100644 --- a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/messages/JsonFriendlyRecord.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/JsonFriendlyRecord.java @@ -12,9 +12,7 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package com.amazonaws.services.kinesis.multilang.messages; - -import java.time.Instant; +package software.amazon.kinesis.multilang.messages; import com.fasterxml.jackson.annotation.JsonProperty; @@ -25,7 +23,6 @@ import lombok.NoArgsConstructor; import lombok.NonNull; import lombok.Setter; import lombok.ToString; -import lombok.experimental.Accessors; import software.amazon.kinesis.retrieval.KinesisClientRecord; /** @@ -42,15 +39,25 @@ public class JsonFriendlyRecord { private byte[] data; private String partitionKey; private String sequenceNumber; - private Instant approximateArrivalTimestamp; + private Long approximateArrivalTimestamp; private Long subSequenceNumber; public static String ACTION = "record"; public static JsonFriendlyRecord fromKinesisClientRecord(@NonNull final KinesisClientRecord record) { - byte[] data = record.data() == null ? null : record.data().array(); + byte[] data; + if (record.data() == null) { + data = null; + } else if (record.data().hasArray()) { + data = record.data().array(); + } else { + data = new byte[record.data().limit()]; + record.data().get(data); + } + Long approximateArrival = record.approximateArrivalTimestamp() == null ? null + : record.approximateArrivalTimestamp().toEpochMilli(); return new JsonFriendlyRecord(data, record.partitionKey(), record.sequenceNumber(), - record.approximateArrivalTimestamp(), record.subSequenceNumber()); + approximateArrival, record.subSequenceNumber()); } @JsonProperty diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/LeaseLostMessage.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/LeaseLostMessage.java new file mode 100644 index 00000000..5afd879b --- /dev/null +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/LeaseLostMessage.java @@ -0,0 +1,25 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.kinesis.multilang.messages; + +/** + * Used to indicate to the client that the record process has lost its lease. + */ +public class LeaseLostMessage extends Message { + + public static final String ACTION = "leaseLost"; + +} diff --git a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/messages/Message.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/Message.java similarity index 88% rename from amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/messages/Message.java rename to amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/Message.java index 3c312b0b..3688ad7b 100644 --- a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/messages/Message.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/Message.java @@ -12,7 +12,7 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package com.amazonaws.services.kinesis.multilang.messages; +package software.amazon.kinesis.multilang.messages; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; @@ -30,10 +30,12 @@ import com.fasterxml.jackson.databind.ObjectMapper; @Type(value = ShutdownMessage.class, name = ShutdownMessage.ACTION), @Type(value = StatusMessage.class, name = StatusMessage.ACTION), @Type(value = ShutdownRequestedMessage.class, name = ShutdownRequestedMessage.ACTION), + @Type(value = LeaseLostMessage.class, name = LeaseLostMessage.ACTION), + @Type(value = ShardEndedMessage.class, name = ShardEndedMessage.ACTION), }) public abstract class Message { - private ObjectMapper mapper = new ObjectMapper();; + private ObjectMapper mapper = new ObjectMapper(); /** * Default constructor. diff --git a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/messages/ProcessRecordsMessage.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/ProcessRecordsMessage.java similarity index 97% rename from amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/messages/ProcessRecordsMessage.java rename to amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/ProcessRecordsMessage.java index e63672ff..0c7441b7 100644 --- a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/messages/ProcessRecordsMessage.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/ProcessRecordsMessage.java @@ -12,7 +12,7 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package com.amazonaws.services.kinesis.multilang.messages; +package software.amazon.kinesis.multilang.messages; import java.util.ArrayList; import java.util.List; diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/ShardEndedMessage.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/ShardEndedMessage.java new file mode 100644 index 00000000..8860d1c3 --- /dev/null +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/ShardEndedMessage.java @@ -0,0 +1,23 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.kinesis.multilang.messages; + +/** + * Used to indicate to the client that the shard has ended. + */ +public class ShardEndedMessage extends Message { + public static final String ACTION = "shardEnded"; +} diff --git a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/messages/ShutdownMessage.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/ShutdownMessage.java similarity index 52% rename from amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/messages/ShutdownMessage.java rename to amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/ShutdownMessage.java index b2b49e3c..5e161661 100644 --- a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/messages/ShutdownMessage.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/ShutdownMessage.java @@ -1,23 +1,22 @@ /* - * Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package com.amazonaws.services.kinesis.multilang.messages; +package software.amazon.kinesis.multilang.messages; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; -import lombok.experimental.Accessors; import software.amazon.kinesis.lifecycle.ShutdownReason; /** diff --git a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/messages/ShutdownRequestedMessage.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/ShutdownRequestedMessage.java similarity index 93% rename from amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/messages/ShutdownRequestedMessage.java rename to amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/ShutdownRequestedMessage.java index 941a8f7e..4c7657f8 100644 --- a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/messages/ShutdownRequestedMessage.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/ShutdownRequestedMessage.java @@ -12,7 +12,7 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package com.amazonaws.services.kinesis.multilang.messages; +package software.amazon.kinesis.multilang.messages; import lombok.NoArgsConstructor; diff --git a/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/StatusMessage.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/StatusMessage.java new file mode 100644 index 00000000..d2acd4b4 --- /dev/null +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/StatusMessage.java @@ -0,0 +1,39 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package software.amazon.kinesis.multilang.messages; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +/** + * A message sent by the client's process to indicate to the record processor that it completed a particular action. + */ +@NoArgsConstructor +@AllArgsConstructor +@Getter +@Setter +public class StatusMessage extends Message { + /** + * The name used for the action field in {@link Message}. + */ + public static final String ACTION = "status"; + + /** + * The name of the most recently received action. + */ + private String responseFor; +} diff --git a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/package-info.java b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/package-info.java similarity index 72% rename from amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/package-info.java rename to amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/package-info.java index 2ec96e40..1f699017 100644 --- a/amazon-kinesis-client-multilang/src/main/java/com/amazonaws/services/kinesis/multilang/package-info.java +++ b/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/package-info.java @@ -1,20 +1,20 @@ /* - * Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ /** * This package provides a KCL application which implements the multi language protocol. The multi language protocol - * defines a system for communication between a KCL multi-lang application and another process (referred to as the + * defines a system for communication between a KCL multi-lang application and another process (referred to as the * "child process") over STDIN and STDOUT of the child process. The units of communication are JSON messages which * represent the actions the receiving entity should perform. The child process is responsible for reacting * appropriately to four different messages: initialize, processRecords, checkpoint, and shutdown. The KCL multi-lang @@ -39,8 +39,15 @@ * "error" : "<NameOfException>" * } * - * { "action" : "shutdown", - * "reason" : "<TERMINATE|ZOMBIE>" + * { "action" : "leaseLost", + * } + * + * { "action" : "shardEnded", + * "checkpoint" : "<SHARD_END>", + * } + * + * { "action" : "shutdownRequested", + * "checkpoint" : "<sequence number>" * } * * @@ -93,13 +100,29 @@ *
  • Begin reading line from STDIN to receive next action
    • * * - *

    Shutdown

    + *

    LeaseLost

    * *
      - *
    1. Read a "shutdown" action from STDIN
      2. - *
    2. Perform shutdown tasks (you may write a checkpoint message at any time)
      3. + *
    3. Read a "leaseLost" action from STDIN
      4. + *
    4. Perform lease lost tasks (you will not be able to checkpoint at this time, since the worker doesn't hold the + * lease)
      5. + *
    5. Write "status" message to STDOUT to indicate you are done.
      6. + *
    + * + *

    ShardEnded

    + * + *
      + *
    1. Read a "shardEnded" action from STDIN
      2. + *
    2. Perform shutdown tasks (you should write a checkpoint message at any time)
      3. + *
    3. Write "status" message to STDOUT to indicate you are done.
      4. + *
    + * + *

    ShutdownRequested

    + * + *
      + *
    1. Read a "shutdownRequested" action from STDIN
      2. + *
    2. Perform shutdown requested related tasks (you may write a checkpoint message at any time)
      3. *
    3. Write "status" message to STDOUT to indicate you are done.
      4. - *
    4. Begin reading line from STDIN to receive next action
      5. *
    * *

    Checkpoint

    @@ -121,5 +144,5 @@ * href="https://docs.python.org/2/library/base64.html">base64 module. * */ -package com.amazonaws.services.kinesis.multilang; +package software.amazon.kinesis.multilang; diff --git a/amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/MultiLangDaemonTest.java b/amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/MultiLangDaemonTest.java deleted file mode 100644 index 92271e2e..00000000 --- a/amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/MultiLangDaemonTest.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/asl/ - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ -package com.amazonaws.services.kinesis.multilang; - -import java.io.PrintStream; -import java.util.concurrent.Executors; - -import org.junit.Test; -import org.mockito.Mockito; - -import software.amazon.awssdk.auth.credentials.AwsCredentials; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.kinesis.coordinator.KinesisClientLibConfiguration; - -public class MultiLangDaemonTest { - - @Test - public void buildWorkerTest() { - // Mocking Kinesis creds - AwsCredentialsProvider provider = Mockito.mock(AwsCredentialsProvider.class); - Mockito.doReturn(Mockito.mock(AwsCredentials.class)).when(provider).resolveCredentials(); - KinesisClientLibConfiguration configuration = new KinesisClientLibConfiguration("Derp", "Blurp", provider, - "Worker"); - - MultiLangRecordProcessorFactory factory = Mockito.mock(MultiLangRecordProcessorFactory.class); - Mockito.doReturn(new String[] { "someExecutableName" }).when(factory).getCommandArray(); - MultiLangDaemon daemon = new MultiLangDaemon(configuration, factory, Executors.newCachedThreadPool()); - } - - @Test - public void usageTest() { - PrintStream printStream = Mockito.mock(PrintStream.class); - - String message = "Everything blew up"; - - MultiLangDaemon.printUsage(printStream, message); - Mockito.verify(printStream, Mockito.times(1)).println(Mockito.contains(message)); - } -} diff --git a/amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoderTest.java b/amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoderTest.java deleted file mode 100644 index 43b507d9..00000000 --- a/amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoderTest.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/asl/ - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ -package com.amazonaws.services.kinesis.multilang.config; - -import static org.junit.Assert.assertEquals; - -import org.junit.Test; - -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.credentials.AwsCredentials; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain; - -public class AWSCredentialsProviderPropertyValueDecoderTest { - - private static final String TEST_ACCESS_KEY_ID = "123"; - private static final String TEST_SECRET_KEY = "456"; - - private String credentialName1 = "com.amazonaws.services.kinesis.multilang.config.AWSCredentialsProviderPropertyValueDecoderTest$AlwaysSucceedCredentialsProvider"; - private String credentialName2 = "com.amazonaws.services.kinesis.multilang.config.AWSCredentialsProviderPropertyValueDecoderTest$ConstructorCredentialsProvider"; - private AWSCredentialsProviderPropertyValueDecoder decoder = new AWSCredentialsProviderPropertyValueDecoder(); - - @Test - public void testSingleProvider() { - AwsCredentialsProvider provider = decoder.decodeValue(credentialName1); - assertEquals(provider.getClass(), AwsCredentialsProviderChain.class); - assertEquals(provider.resolveCredentials().accessKeyId(), TEST_ACCESS_KEY_ID); - assertEquals(provider.resolveCredentials().secretAccessKey(), TEST_SECRET_KEY); - } - - @Test - public void testTwoProviders() { - AwsCredentialsProvider provider = decoder.decodeValue(credentialName1 + "," + credentialName1); - assertEquals(provider.getClass(), AwsCredentialsProviderChain.class); - assertEquals(provider.resolveCredentials().accessKeyId(), TEST_ACCESS_KEY_ID); - assertEquals(provider.resolveCredentials().secretAccessKey(), TEST_SECRET_KEY); - } - - @Test - public void testProfileProviderWithOneArg() { - AwsCredentialsProvider provider = decoder.decodeValue(credentialName2 + "|arg"); - assertEquals(provider.getClass(), AwsCredentialsProviderChain.class); - assertEquals(provider.resolveCredentials().accessKeyId(), "arg"); - assertEquals(provider.resolveCredentials().secretAccessKey(), "blank"); - } - - @Test - public void testProfileProviderWithTwoArgs() { - AwsCredentialsProvider provider = decoder.decodeValue(credentialName2 + "|arg1|arg2"); - assertEquals(provider.getClass(), AwsCredentialsProviderChain.class); - assertEquals(provider.resolveCredentials().accessKeyId(), "arg1"); - assertEquals(provider.resolveCredentials().secretAccessKey(), "arg2"); - } - - /** - * This credentials provider will always succeed - */ - public static class AlwaysSucceedCredentialsProvider implements AwsCredentialsProvider { - - @Override - public AwsCredentials resolveCredentials() { - return AwsBasicCredentials.create(TEST_ACCESS_KEY_ID, TEST_SECRET_KEY); - } - - } - - /** - * This credentials provider needs a constructor call to instantiate it - */ - public static class ConstructorCredentialsProvider implements AwsCredentialsProvider { - - private String arg1; - private String arg2; - - public ConstructorCredentialsProvider(String arg1) { - this.arg1 = arg1; - this.arg2 = "blank"; - } - - public ConstructorCredentialsProvider(String arg1, String arg2) { - this.arg1 = arg1; - this.arg2 = arg2; - } - - @Override - public AwsCredentials resolveCredentials() { - return AwsBasicCredentials.create(arg1, arg2); - } - - } -} diff --git a/amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/Matchers.java b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/Matchers.java similarity index 98% rename from amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/Matchers.java rename to amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/Matchers.java index 6ec8962a..f24d21a4 100644 --- a/amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/Matchers.java +++ b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/Matchers.java @@ -12,7 +12,7 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package com.amazonaws.services.kinesis.multilang; +package software.amazon.kinesis.multilang; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.nullValue; diff --git a/amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/MessageReaderTest.java b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MessageReaderTest.java similarity index 90% rename from amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/MessageReaderTest.java rename to amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MessageReaderTest.java index 89ca0d17..b9ba2a30 100644 --- a/amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/MessageReaderTest.java +++ b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MessageReaderTest.java @@ -1,18 +1,18 @@ /* - * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package com.amazonaws.services.kinesis.multilang; +package software.amazon.kinesis.multilang; import java.io.BufferedReader; import java.io.ByteArrayInputStream; @@ -23,14 +23,14 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import com.amazonaws.services.kinesis.multilang.messages.Message; -import com.amazonaws.services.kinesis.multilang.messages.StatusMessage; +import software.amazon.kinesis.multilang.MessageReader; +import software.amazon.kinesis.multilang.messages.Message; +import software.amazon.kinesis.multilang.messages.StatusMessage; import com.fasterxml.jackson.databind.ObjectMapper; public class MessageReaderTest { diff --git a/amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/MessageWriterTest.java b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MessageWriterTest.java similarity index 73% rename from amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/MessageWriterTest.java rename to amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MessageWriterTest.java index 22a448b1..1fd21dee 100644 --- a/amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/MessageWriterTest.java +++ b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MessageWriterTest.java @@ -12,7 +12,7 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package com.amazonaws.services.kinesis.multilang; +package software.amazon.kinesis.multilang; import java.io.IOException; import java.io.OutputStream; @@ -25,10 +25,16 @@ import java.util.concurrent.Future; import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import org.mockito.Mockito; -import com.amazonaws.services.kinesis.multilang.messages.Message; +import software.amazon.kinesis.lifecycle.events.LeaseLostInput; +import software.amazon.kinesis.lifecycle.events.ShardEndedInput; +import software.amazon.kinesis.multilang.MessageWriter; +import software.amazon.kinesis.multilang.messages.LeaseLostMessage; +import software.amazon.kinesis.multilang.messages.Message; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -37,12 +43,17 @@ import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput; import software.amazon.kinesis.lifecycle.ShutdownReason; import software.amazon.kinesis.retrieval.KinesisClientRecord; +import static org.mockito.Mockito.verify; + public class MessageWriterTest { private static final String shardId = "shard-123"; MessageWriter messageWriter; OutputStream stream; + @Rule + public final ExpectedException thrown = ExpectedException.none(); + // ExecutorService executor; @Before @@ -59,27 +70,27 @@ public class MessageWriterTest { public void writeCheckpointMessageNoErrorTest() throws IOException, InterruptedException, ExecutionException { Future future = this.messageWriter.writeCheckpointMessageWithError("1234", 0L, null); future.get(); - Mockito.verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), + verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt()); - Mockito.verify(this.stream, Mockito.atLeastOnce()).flush(); + verify(this.stream, Mockito.atLeastOnce()).flush(); } @Test public void writeCheckpointMessageWithErrorTest() throws IOException, InterruptedException, ExecutionException { Future future = this.messageWriter.writeCheckpointMessageWithError("1234", 0L, new Throwable()); future.get(); - Mockito.verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), + verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt()); - Mockito.verify(this.stream, Mockito.atLeastOnce()).flush(); + verify(this.stream, Mockito.atLeastOnce()).flush(); } @Test public void writeInitializeMessageTest() throws IOException, InterruptedException, ExecutionException { Future future = this.messageWriter.writeInitializeMessage(InitializationInput.builder().shardId(shardId).build()); future.get(); - Mockito.verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), + verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt()); - Mockito.verify(this.stream, Mockito.atLeastOnce()).flush(); + verify(this.stream, Mockito.atLeastOnce()).flush(); } @Test @@ -92,19 +103,19 @@ public class MessageWriterTest { Future future = this.messageWriter.writeProcessRecordsMessage(ProcessRecordsInput.builder().records(records).build()); future.get(); - Mockito.verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), + verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt()); - Mockito.verify(this.stream, Mockito.atLeastOnce()).flush(); + verify(this.stream, Mockito.atLeastOnce()).flush(); } @Test public void writeShutdownMessageTest() throws IOException, InterruptedException, ExecutionException { - Future future = this.messageWriter.writeShutdownMessage(ShutdownReason.SHARD_END); + Future future = this.messageWriter.writeShardEndedMessage(ShardEndedInput.builder().build()); future.get(); - Mockito.verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), + verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt()); - Mockito.verify(this.stream, Mockito.atLeastOnce()).flush(); + verify(this.stream, Mockito.atLeastOnce()).flush(); } @Test @@ -112,9 +123,9 @@ public class MessageWriterTest { Future future = this.messageWriter.writeShutdownRequestedMessage(); future.get(); - Mockito.verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), + verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt()); - Mockito.verify(this.stream, Mockito.atLeastOnce()).flush(); + verify(this.stream, Mockito.atLeastOnce()).flush(); } @Test @@ -128,25 +139,21 @@ public class MessageWriterTest { @Test public void objectMapperFails() throws JsonProcessingException, InterruptedException, ExecutionException { + thrown.expect(RuntimeException.class); + thrown.expectMessage("Encountered I/O error while writing LeaseLostMessage action to subprocess"); + ObjectMapper mapper = Mockito.mock(ObjectMapper.class); Mockito.doThrow(JsonProcessingException.class).when(mapper).writeValueAsString(Mockito.any(Message.class)); messageWriter = new MessageWriter().initialize(stream, shardId, mapper, Executors.newCachedThreadPool()); - try { - messageWriter.writeShutdownMessage(ShutdownReason.LEASE_LOST); - Assert.fail("The mapper failed so no write method should be able to succeed."); - } catch (Exception e) { - // Note that this is different than the stream failing. The stream is expected to fail, so we handle it - // gracefully, but the JSON mapping should always succeed. - } - + messageWriter.writeLeaseLossMessage(LeaseLostInput.builder().build()); } @Test public void closeWriterTest() throws IOException { Assert.assertTrue(this.messageWriter.isOpen()); this.messageWriter.close(); - Mockito.verify(this.stream, Mockito.times(1)).close(); + verify(this.stream, Mockito.times(1)).close(); Assert.assertFalse(this.messageWriter.isOpen()); try { // Any message should fail diff --git a/amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/MultiLangDaemonConfigTest.java b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MultiLangDaemonConfigTest.java similarity index 78% rename from amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/MultiLangDaemonConfigTest.java rename to amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MultiLangDaemonConfigTest.java index a08f6673..e9777c5e 100644 --- a/amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/MultiLangDaemonConfigTest.java +++ b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MultiLangDaemonConfigTest.java @@ -12,7 +12,7 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package com.amazonaws.services.kinesis.multilang; +package software.amazon.kinesis.multilang; import static org.junit.Assert.assertNotNull; import static org.mockito.Matchers.any; @@ -22,20 +22,20 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.util.Properties; +import org.apache.commons.beanutils.BeanUtilsBean; +import org.apache.commons.beanutils.ConvertUtilsBean; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.runners.MockitoJUnitRunner; -import com.amazonaws.services.kinesis.multilang.config.KinesisClientLibConfigurator; - import junit.framework.Assert; import software.amazon.awssdk.auth.credentials.AwsCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.kinesis.coordinator.KinesisClientLibConfiguration; +import software.amazon.kinesis.multilang.config.KinesisClientLibConfigurator; +import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration; @RunWith(MockitoJUnitRunner.class) public class MultiLangDaemonConfigTest { @@ -50,14 +50,18 @@ public class MultiLangDaemonConfigTest { @Before public void setup() { + ConvertUtilsBean convertUtilsBean = new ConvertUtilsBean(); + BeanUtilsBean utilsBean = new BeanUtilsBean(convertUtilsBean); + MultiLangDaemonConfiguration multiLangDaemonConfiguration = new MultiLangDaemonConfiguration(utilsBean, + convertUtilsBean); + multiLangDaemonConfiguration.setApplicationName("cool-app"); + multiLangDaemonConfiguration.setStreamName("cool-stream"); + multiLangDaemonConfiguration.setWorkerIdentifier("cool-worker"); when(credentialsProvider.resolveCredentials()).thenReturn(creds); when(creds.accessKeyId()).thenReturn("cool-user"); - when(configurator.getConfiguration(any(Properties.class))).thenReturn( - new KinesisClientLibConfiguration("cool-app", "cool-stream", credentialsProvider, "cool-worker")); + when(configurator.getConfiguration(any(Properties.class))).thenReturn(multiLangDaemonConfiguration); } - // TODO: Fix test - @Ignore @Test public void constructorTest() throws IOException { String PROPERTIES = "executableName = randomEXE \n" + "applicationName = testApp \n" @@ -71,12 +75,10 @@ public class MultiLangDaemonConfigTest { MultiLangDaemonConfig deamonConfig = new MultiLangDaemonConfig(FILENAME, classLoader, configurator); assertNotNull(deamonConfig.getExecutorService()); - assertNotNull(deamonConfig.getKinesisClientLibConfiguration()); + assertNotNull(deamonConfig.getMultiLangDaemonConfiguration()); assertNotNull(deamonConfig.getRecordProcessorFactory()); } - // TODO: Fix test - @Ignore @Test public void propertyValidation() { String PROPERTIES_NO_EXECUTABLE_NAME = "applicationName = testApp \n" + "streamName = fakeStream \n" diff --git a/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MultiLangDaemonTest.java b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MultiLangDaemonTest.java new file mode 100644 index 00000000..b3f92be2 --- /dev/null +++ b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MultiLangDaemonTest.java @@ -0,0 +1,276 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package software.amazon.kinesis.multilang; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.isEmptyOrNullString; +import static org.hamcrest.Matchers.nullValue; +import static org.junit.Assert.assertThat; +import static org.mockito.Matchers.anyObject; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; +import org.slf4j.LoggerFactory; + +import ch.qos.logback.classic.LoggerContext; +import ch.qos.logback.classic.joran.JoranConfigurator; +import software.amazon.kinesis.coordinator.Scheduler; +import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration; + +@RunWith(MockitoJUnitRunner.class) +public class MultiLangDaemonTest { + @Mock + private Scheduler scheduler; + @Mock + private MultiLangDaemonConfig config; + @Mock + private ExecutorService executorService; + @Mock + private Future futureInteger; + @Mock + private MultiLangDaemonConfiguration multiLangDaemonConfiguration; + @Mock + private Runtime runtime; + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + @Rule + public final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private MultiLangDaemon daemon; + + @Before + public void setup() { + daemon = new MultiLangDaemon() { + @Override + Scheduler buildScheduler(final MultiLangDaemonConfig configuration) { + return scheduler; + } + }; + } + + @Test + public void testSuccessfulNoOptionsJCommanderBuild() { + String testPropertiesFile = "/test/properties/file"; + MultiLangDaemon.MultiLangDaemonArguments arguments = new MultiLangDaemon.MultiLangDaemonArguments(); + daemon.buildJCommanderAndParseArgs(arguments, new String[] { testPropertiesFile }); + + assertThat(arguments.propertiesFile, nullValue()); + assertThat(arguments.logConfiguration, nullValue()); + assertThat(arguments.parameters.size(), equalTo(1)); + assertThat(arguments.parameters.get(0), equalTo(testPropertiesFile)); + } + + @Test + public void testSuccessfulOptionsJCommanderBuild() { + String propertiesOption = "/test/properties/file/option"; + String propertiesFileArgs = "/test/properties/args"; + String[] args = new String[] { "-p", propertiesOption, propertiesFileArgs }; + MultiLangDaemon.MultiLangDaemonArguments arguments = new MultiLangDaemon.MultiLangDaemonArguments(); + daemon.buildJCommanderAndParseArgs(arguments, args); + + assertThat(arguments.propertiesFile, equalTo(propertiesOption)); + assertThat(arguments.logConfiguration, nullValue()); + assertThat(arguments.parameters.size(), equalTo(1)); + assertThat(arguments.parameters.get(0), equalTo(propertiesFileArgs)); + } + + @Test + public void testEmptyArgsJCommanderBuild() { + MultiLangDaemon.MultiLangDaemonArguments arguments = new MultiLangDaemon.MultiLangDaemonArguments(); + String[] args = new String[] {}; + daemon.buildJCommanderAndParseArgs(arguments, args); + + assertThat(arguments.propertiesFile, nullValue()); + assertThat(arguments.logConfiguration, nullValue()); + assertThat(arguments.parameters, empty()); + } + + @Test + public void testSuccessfulLoggingConfiguration() { + LoggerContext loggerContext = spy((LoggerContext) LoggerFactory.getILoggerFactory()); + JoranConfigurator configurator = spy(new JoranConfigurator()); + + String logConfiguration = this.getClass().getClassLoader().getResource("logback.xml").getPath(); + daemon.configureLogging(logConfiguration, loggerContext, configurator); + + verify(loggerContext).reset(); + verify(configurator).setContext(eq(loggerContext)); + } + + @Test + public void testUnsuccessfulLoggingConfiguration() { + LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory(); + JoranConfigurator configurator = new JoranConfigurator(); + + expectedException.expect(RuntimeException.class); + expectedException.expectMessage(containsString("Error while loading log configuration:")); + + String logConfiguration = "blahblahblah"; + + daemon.configureLogging(logConfiguration, loggerContext, configurator); + } + + @Test + public void testNoPropertiesFileArgumentOrOption() { + expectedException.expect(RuntimeException.class); + expectedException.expectMessage(equalTo("Properties file missing, please provide a properties file")); + + MultiLangDaemon.MultiLangDaemonArguments arguments = new MultiLangDaemon.MultiLangDaemonArguments(); + + daemon.propertiesFile(arguments); + } + + @Test + public void testSuccessfulPropertiesArgument() { + String expectedPropertiesFile = "/test/properties/file"; + MultiLangDaemon.MultiLangDaemonArguments arguments = new MultiLangDaemon.MultiLangDaemonArguments(); + arguments.parameters = Collections.singletonList(expectedPropertiesFile); + + String propertiesFile = daemon.propertiesFile(arguments); + + assertThat(propertiesFile, equalTo(expectedPropertiesFile)); + } + + @Test + public void testPropertiesOptionsOverrideArgument() { + String propertiesArgument = "/test/properties/argument"; + String propertiesOptions = "/test/properties/options"; + + MultiLangDaemon.MultiLangDaemonArguments arguments = new MultiLangDaemon.MultiLangDaemonArguments(); + arguments.parameters = Collections.singletonList(propertiesArgument); + arguments.propertiesFile = propertiesOptions; + + String propertiesFile = daemon.propertiesFile(arguments); + + assertThat(propertiesFile, equalTo(propertiesOptions)); + } + + @Test + public void testExtraArgumentsFailure() { + expectedException.expect(RuntimeException.class); + expectedException.expectMessage(containsString("Expected a single argument, but found multiple arguments.")); + + MultiLangDaemon.MultiLangDaemonArguments arguments = new MultiLangDaemon.MultiLangDaemonArguments(); + arguments.parameters = Arrays.asList("parameter1", "parameter2"); + + daemon.propertiesFile(arguments); + } + + @Test + public void testBuildMultiLangConfigMissingPropertiesFile() { + expectedException.expect(RuntimeException.class); + expectedException.expectMessage(containsString("Error while reading properties file:")); + + daemon.buildMultiLangDaemonConfig("blahblahblah"); + } + + @Test + public void testBuildMultiLangConfigWithIncorrectInformation() throws IOException { + File propertiesFile = temporaryFolder.newFile("temp.properties"); + + expectedException.expect(RuntimeException.class); + expectedException.expectMessage(containsString("Must provide an executable name in the properties file")); + + daemon.buildMultiLangDaemonConfig(propertiesFile.getAbsolutePath()); + } + + @Test + public void testSuccessfulSubmitRunnerAndWait() throws Exception { + int expectedExitCode = 0; + + MultiLangDaemon.MultiLangRunner runner = new MultiLangDaemon.MultiLangRunner(scheduler); + when(config.getExecutorService()).thenReturn(executorService); + when(executorService.submit(eq(runner))).thenReturn(futureInteger); + when(futureInteger.get()).thenReturn(expectedExitCode); + + int exitCode = daemon.submitRunnerAndWait(config, runner); + + assertThat(exitCode, equalTo(expectedExitCode)); + } + + @Test + public void testErrorSubmitRunnerAndWait() throws Exception { + int expectedExitCode = 1; + + MultiLangDaemon.MultiLangRunner runner = new MultiLangDaemon.MultiLangRunner(scheduler); + when(config.getExecutorService()).thenReturn(executorService); + when(executorService.submit(eq(runner))).thenReturn(futureInteger); + when(futureInteger.get()).thenThrow(ExecutionException.class); + + int exitCode = daemon.submitRunnerAndWait(config, runner); + + assertThat(exitCode, equalTo(expectedExitCode)); + } + + @Test + public void testSetupShutdownHook() { + when(config.getMultiLangDaemonConfiguration()).thenReturn(multiLangDaemonConfiguration); + when(multiLangDaemonConfiguration.getShutdownGraceMillis()).thenReturn(1000L); + doNothing().when(runtime).addShutdownHook(anyObject()); + + MultiLangDaemon.MultiLangRunner runner = new MultiLangDaemon.MultiLangRunner(scheduler); + daemon.setupShutdownHook(runtime, runner, config); + + verify(multiLangDaemonConfiguration).getShutdownGraceMillis(); + verify(runtime).addShutdownHook(anyObject()); + } + + @Test + public void testSuccessfulRunner() throws Exception { + MultiLangDaemon.MultiLangRunner runner = new MultiLangDaemon.MultiLangRunner(scheduler); + doNothing().when(scheduler).run(); + + int exit = runner.call(); + + assertThat(exit, equalTo(0)); + + verify(scheduler).run(); + } + + @Test + public void testUnsuccessfulRunner() throws Exception { + MultiLangDaemon.MultiLangRunner runner = new MultiLangDaemon.MultiLangRunner(scheduler); + doThrow(Exception.class).when(scheduler).run(); + + int exit = runner.call(); + + assertThat(exit, equalTo(1)); + + verify(scheduler).run(); + } +} diff --git a/amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/MultiLangProtocolTest.java b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MultiLangProtocolTest.java similarity index 85% rename from amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/MultiLangProtocolTest.java rename to amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MultiLangProtocolTest.java index 5e51cc05..813479e1 100644 --- a/amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/MultiLangProtocolTest.java +++ b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MultiLangProtocolTest.java @@ -12,7 +12,7 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package com.amazonaws.services.kinesis.multilang; +package software.amazon.kinesis.multilang; import static org.hamcrest.CoreMatchers.equalTo; import static org.junit.Assert.assertThat; @@ -50,10 +50,15 @@ import software.amazon.kinesis.exceptions.InvalidStateException; import software.amazon.kinesis.exceptions.KinesisClientLibDependencyException; import software.amazon.kinesis.exceptions.ShutdownException; import software.amazon.kinesis.exceptions.ThrottlingException; -import com.amazonaws.services.kinesis.multilang.messages.CheckpointMessage; -import com.amazonaws.services.kinesis.multilang.messages.Message; -import com.amazonaws.services.kinesis.multilang.messages.ProcessRecordsMessage; -import com.amazonaws.services.kinesis.multilang.messages.StatusMessage; +import software.amazon.kinesis.lifecycle.events.LeaseLostInput; +import software.amazon.kinesis.lifecycle.events.ShardEndedInput; +import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration; +import software.amazon.kinesis.multilang.messages.CheckpointMessage; +import software.amazon.kinesis.multilang.messages.LeaseLostMessage; +import software.amazon.kinesis.multilang.messages.Message; +import software.amazon.kinesis.multilang.messages.ProcessRecordsMessage; +import software.amazon.kinesis.multilang.messages.ShardEndedMessage; +import software.amazon.kinesis.multilang.messages.StatusMessage; import com.google.common.util.concurrent.SettableFuture; import software.amazon.kinesis.coordinator.KinesisClientLibConfiguration; @@ -77,7 +82,7 @@ public class MultiLangProtocolTest { @Mock private RecordProcessorCheckpointer checkpointer; @Mock - private KinesisClientLibConfiguration configuration; + private MultiLangDaemonConfiguration configuration; @Before public void setup() { @@ -85,7 +90,7 @@ public class MultiLangProtocolTest { protocol = new MultiLangProtocolForTesting(messageReader, messageWriter, InitializationInput.builder().shardId(shardId).build(), configuration); - when(configuration.getTimeoutInSeconds()).thenReturn(Optional.empty()); + when(configuration.getTimeoutInSeconds()).thenReturn(null); } private Future buildFuture(T value) { @@ -121,16 +126,20 @@ public class MultiLangProtocolTest { } @Test - public void shutdownTest() throws InterruptedException, ExecutionException { - when(messageWriter.writeShutdownMessage(any(ShutdownReason.class))).thenReturn(buildFuture(true)); - when(messageReader.getNextMessageFromSTDOUT()).thenReturn(buildFuture( - new StatusMessage("shutdown"), Message.class)); + public void leaseLostTest() { + when(messageWriter.writeLeaseLossMessage(any(LeaseLostInput.class))).thenReturn(buildFuture(true)); + when(messageReader.getNextMessageFromSTDOUT()).thenReturn(buildFuture(new StatusMessage(LeaseLostMessage.ACTION), Message.class)); - Mockito.doReturn(buildFuture(true)).when(messageWriter) - .writeShutdownMessage(any(ShutdownReason.class)); - Mockito.doReturn(buildFuture(new StatusMessage("shutdown"))) - .when(messageReader).getNextMessageFromSTDOUT(); - assertThat(protocol.shutdown(null, ShutdownReason.LEASE_LOST), equalTo(true)); + assertThat(protocol.leaseLost(LeaseLostInput.builder().build()), equalTo(true)); + + } + + @Test + public void shardEndedTest() { + when(messageWriter.writeShardEndedMessage(any(ShardEndedInput.class))).thenReturn(buildFuture(true)); + when(messageReader.getNextMessageFromSTDOUT()).thenReturn(buildFuture(new StatusMessage(ShardEndedMessage.ACTION))); + + assertThat(protocol.shardEnded(ShardEndedInput.builder().build()), equalTo(true)); } @Test @@ -215,7 +224,7 @@ public class MultiLangProtocolTest { when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class))).thenReturn(buildFuture(true)); Future future = Mockito.mock(Future.class); when(messageReader.getNextMessageFromSTDOUT()).thenReturn(future); - when(configuration.getTimeoutInSeconds()).thenReturn(Optional.of(5)); + when(configuration.getTimeoutInSeconds()).thenReturn(5); when(future.get(anyInt(), eq(TimeUnit.SECONDS))).thenThrow(TimeoutException.class); protocol = new MultiLangProtocolForTesting(messageReader, messageWriter, @@ -230,7 +239,7 @@ public class MultiLangProtocolTest { when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class))).thenReturn(buildFuture(true)); when(messageReader.getNextMessageFromSTDOUT()).thenReturn(buildFuture( new StatusMessage("processRecords"), Message.class)); - when(configuration.getTimeoutInSeconds()).thenReturn(Optional.of(5)); + when(configuration.getTimeoutInSeconds()).thenReturn(5); assertTrue(protocol.processRecords(ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST).build())); } @@ -247,7 +256,7 @@ public class MultiLangProtocolTest { MultiLangProtocolForTesting(final MessageReader messageReader, final MessageWriter messageWriter, final InitializationInput initializationInput, - final KinesisClientLibConfiguration configuration) { + final MultiLangDaemonConfiguration configuration) { super(messageReader, messageWriter, initializationInput, configuration); } diff --git a/amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/ReadSTDERRTaskTest.java b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/ReadSTDERRTaskTest.java similarity index 79% rename from amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/ReadSTDERRTaskTest.java rename to amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/ReadSTDERRTaskTest.java index 55e96365..96e94588 100644 --- a/amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/ReadSTDERRTaskTest.java +++ b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/ReadSTDERRTaskTest.java @@ -1,18 +1,18 @@ /* - * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. * - * Licensed under the Amazon Software License (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at * - * http://aws.amazon.com/asl/ + * http://aws.amazon.com/asl/ * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package com.amazonaws.services.kinesis.multilang; +package software.amazon.kinesis.multilang; import java.io.BufferedReader; import java.io.ByteArrayInputStream; @@ -27,6 +27,8 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; +import software.amazon.kinesis.multilang.DrainChildSTDERRTask; +import software.amazon.kinesis.multilang.LineReaderTask; public class ReadSTDERRTaskTest { diff --git a/amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/StreamingShardRecordProcessorFactoryTest.java b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/StreamingShardRecordProcessorFactoryTest.java similarity index 81% rename from amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/StreamingShardRecordProcessorFactoryTest.java rename to amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/StreamingShardRecordProcessorFactoryTest.java index 2eba9833..139f5d25 100644 --- a/amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/StreamingShardRecordProcessorFactoryTest.java +++ b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/StreamingShardRecordProcessorFactoryTest.java @@ -12,12 +12,15 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package com.amazonaws.services.kinesis.multilang; +package software.amazon.kinesis.multilang; import software.amazon.kinesis.coordinator.KinesisClientLibConfiguration; import org.junit.Assert; import org.junit.Test; +import software.amazon.kinesis.multilang.MultiLangRecordProcessorFactory; +import software.amazon.kinesis.multilang.MultiLangShardRecordProcessor; +import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration; import software.amazon.kinesis.processor.ShardRecordProcessor; import org.junit.runner.RunWith; import org.mockito.Mock; @@ -27,7 +30,7 @@ import org.mockito.runners.MockitoJUnitRunner; public class StreamingShardRecordProcessorFactoryTest { @Mock - private KinesisClientLibConfiguration configuration; + private MultiLangDaemonConfiguration configuration; @Test public void createProcessorTest() { diff --git a/amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/StreamingShardRecordProcessorTest.java b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/StreamingShardRecordProcessorTest.java similarity index 89% rename from amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/StreamingShardRecordProcessorTest.java rename to amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/StreamingShardRecordProcessorTest.java index e51bc2a1..67189b72 100644 --- a/amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/StreamingShardRecordProcessorTest.java +++ b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/StreamingShardRecordProcessorTest.java @@ -12,48 +12,7 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package com.amazonaws.services.kinesis.multilang; - -import software.amazon.kinesis.exceptions.InvalidStateException; -import software.amazon.kinesis.exceptions.KinesisClientLibDependencyException; -import software.amazon.kinesis.exceptions.ShutdownException; -import software.amazon.kinesis.exceptions.ThrottlingException; -import software.amazon.awssdk.services.kinesis.model.Record; -import software.amazon.kinesis.processor.Checkpointer; -import software.amazon.kinesis.processor.PreparedCheckpointer; -import software.amazon.kinesis.processor.RecordProcessorCheckpointer; -import software.amazon.kinesis.coordinator.KinesisClientLibConfiguration; -import software.amazon.kinesis.lifecycle.ShutdownReason; -import software.amazon.kinesis.lifecycle.events.InitializationInput; -import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput; -import software.amazon.kinesis.lifecycle.ShutdownInput; -import com.amazonaws.services.kinesis.multilang.messages.InitializeMessage; -import com.amazonaws.services.kinesis.multilang.messages.Message; -import com.amazonaws.services.kinesis.multilang.messages.ProcessRecordsMessage; -import com.amazonaws.services.kinesis.multilang.messages.ShutdownMessage; -import com.amazonaws.services.kinesis.multilang.messages.StatusMessage; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.runners.MockitoJUnitRunner; -import org.mockito.stubbing.Answer; -import software.amazon.kinesis.retrieval.KinesisClientRecord; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; +package software.amazon.kinesis.multilang; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyLong; @@ -64,6 +23,47 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.runners.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import software.amazon.awssdk.services.kinesis.model.Record; +import software.amazon.kinesis.exceptions.InvalidStateException; +import software.amazon.kinesis.exceptions.KinesisClientLibDependencyException; +import software.amazon.kinesis.exceptions.ShutdownException; +import software.amazon.kinesis.exceptions.ThrottlingException; +import software.amazon.kinesis.lifecycle.events.InitializationInput; +import software.amazon.kinesis.lifecycle.events.LeaseLostInput; +import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput; +import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration; +import software.amazon.kinesis.multilang.messages.InitializeMessage; +import software.amazon.kinesis.multilang.messages.Message; +import software.amazon.kinesis.multilang.messages.ProcessRecordsMessage; +import software.amazon.kinesis.multilang.messages.ShutdownMessage; +import software.amazon.kinesis.multilang.messages.StatusMessage; +import software.amazon.kinesis.processor.Checkpointer; +import software.amazon.kinesis.processor.PreparedCheckpointer; +import software.amazon.kinesis.processor.RecordProcessorCheckpointer; +import software.amazon.kinesis.retrieval.KinesisClientRecord; + @RunWith(MockitoJUnitRunner.class) public class StreamingShardRecordProcessorTest { @@ -73,6 +73,8 @@ public class StreamingShardRecordProcessorTest { @Mock private Future messageFuture; + @Mock + private Future trueFuture; private RecordProcessorCheckpointer unimplementedCheckpointer = new RecordProcessorCheckpointer() { @@ -146,12 +148,11 @@ public class StreamingShardRecordProcessorTest { private MultiLangShardRecordProcessor recordProcessor; @Mock - private KinesisClientLibConfiguration configuration; + private MultiLangDaemonConfiguration configuration; @Before public void prepare() throws IOException, InterruptedException, ExecutionException { // Fake command - String command = "derp"; systemExitCount = 0; // Mocks @@ -161,7 +162,7 @@ public class StreamingShardRecordProcessorTest { messageWriter = Mockito.mock(MessageWriter.class); messageReader = Mockito.mock(MessageReader.class); errorReader = Mockito.mock(DrainChildSTDERRTask.class); - when(configuration.getTimeoutInSeconds()).thenReturn(Optional.empty()); + when(configuration.getTimeoutInSeconds()).thenReturn(null); recordProcessor = new MultiLangShardRecordProcessor(new ProcessBuilder(), executor, new ObjectMapper(), messageWriter, @@ -187,13 +188,12 @@ public class StreamingShardRecordProcessorTest { Mockito.doReturn(outputStream).when(process).getOutputStream(); Mockito.doReturn(Mockito.mock(Future.class)).when(messageReader).drainSTDOUT(); - Future trueFuture = Mockito.mock(Future.class); Mockito.doReturn(true).when(trueFuture).get(); when(messageWriter.writeInitializeMessage(any(InitializationInput.class))).thenReturn(trueFuture); when(messageWriter.writeCheckpointMessageWithError(anyString(), anyLong(), any(Throwable.class))).thenReturn(trueFuture); when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class))).thenReturn(trueFuture); - when(messageWriter.writeShutdownMessage(any(ShutdownReason.class))).thenReturn(trueFuture); + when(messageWriter.writeLeaseLossMessage(any(LeaseLostInput.class))).thenReturn(trueFuture); } private void phases(Answer answer) throws InterruptedException, ExecutionException { @@ -215,8 +215,7 @@ public class StreamingShardRecordProcessorTest { .checkpointer(unimplementedCheckpointer).build()); recordProcessor.processRecords(ProcessRecordsInput.builder().records(testRecords) .checkpointer(unimplementedCheckpointer).build()); - recordProcessor.shutdown(ShutdownInput.builder().checkpointer(unimplementedCheckpointer) - .shutdownReason(ShutdownReason.LEASE_LOST).build()); + recordProcessor.leaseLost(LeaseLostInput.builder().build()); } @Test @@ -246,7 +245,7 @@ public class StreamingShardRecordProcessorTest { .writeInitializeMessage(argThat(Matchers.withInit( InitializationInput.builder().shardId(shardId).build()))); verify(messageWriter, times(2)).writeProcessRecordsMessage(any(ProcessRecordsInput.class)); - verify(messageWriter).writeShutdownMessage(ShutdownReason.LEASE_LOST); + verify(messageWriter).writeLeaseLossMessage(any(LeaseLostInput.class)); } @Test @@ -278,7 +277,7 @@ public class StreamingShardRecordProcessorTest { verify(messageWriter).writeInitializeMessage(argThat(Matchers.withInit(InitializationInput.builder() .shardId(shardId).build()))); verify(messageWriter, times(2)).writeProcessRecordsMessage(any(ProcessRecordsInput.class)); - verify(messageWriter, never()).writeShutdownMessage(ShutdownReason.LEASE_LOST); + verify(messageWriter, never()).writeLeaseLossMessage(any(LeaseLostInput.class)); Assert.assertEquals(1, systemExitCount); } } diff --git a/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoderTest.java b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoderTest.java new file mode 100644 index 00000000..5ff395c2 --- /dev/null +++ b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoderTest.java @@ -0,0 +1,167 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package software.amazon.kinesis.multilang.config; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; + +import java.util.Arrays; + +import com.amazonaws.auth.BasicAWSCredentials; +import lombok.ToString; +import org.hamcrest.Description; +import org.hamcrest.Matcher; +import org.hamcrest.TypeSafeDiagnosingMatcher; +import org.junit.Test; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.AWSCredentialsProviderChain; + +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain; + +public class AWSCredentialsProviderPropertyValueDecoderTest { + + private static final String TEST_ACCESS_KEY_ID = "123"; + private static final String TEST_SECRET_KEY = "456"; + + private String credentialName1 = "software.amazon.kinesis.multilang.config.AWSCredentialsProviderPropertyValueDecoderTest$AlwaysSucceedCredentialsProvider"; + private String credentialName2 = "software.amazon.kinesis.multilang.config.AWSCredentialsProviderPropertyValueDecoderTest$ConstructorCredentialsProvider"; + private AWSCredentialsProviderPropertyValueDecoder decoder = new AWSCredentialsProviderPropertyValueDecoder(); + + @ToString + private static class AWSCredentialsMatcher extends TypeSafeDiagnosingMatcher { + + private final Matcher akidMatcher; + private final Matcher secretMatcher; + private final Matcher> classMatcher; + + public AWSCredentialsMatcher(String akid, String secret) { + this.akidMatcher = equalTo(akid); + this.secretMatcher = equalTo(secret); + this.classMatcher = instanceOf(AWSCredentialsProviderChain.class); + } + + private AWSCredentialsMatcher(AWSCredentials expected) { + this(expected.getAWSAccessKeyId(), expected.getAWSSecretKey()); + } + + @Override + protected boolean matchesSafely(AWSCredentialsProvider item, Description mismatchDescription) { + AWSCredentials actual = item.getCredentials(); + boolean matched = true; + + if (!classMatcher.matches(item)) { + classMatcher.describeMismatch(item, mismatchDescription); + matched = false; + } + + if (!akidMatcher.matches(actual.getAWSAccessKeyId())) { + akidMatcher.describeMismatch(actual.getAWSAccessKeyId(), mismatchDescription); + matched = false; + } + if (!secretMatcher.matches(actual.getAWSSecretKey())) { + secretMatcher.describeMismatch(actual.getAWSSecretKey(), mismatchDescription); + matched = false; + } + return matched; + } + + @Override + public void describeTo(Description description) { + description.appendText("An AWSCredentialsProvider that provides an AWSCredential matching: ") + .appendList("(", ", ", ")", Arrays.asList(classMatcher, akidMatcher, secretMatcher)); + } + + } + + private static AWSCredentialsMatcher hasCredentials(String akid, String secret) { + return new AWSCredentialsMatcher(akid, secret); + } + + @Test + public void testSingleProvider() { + AWSCredentialsProvider provider = decoder.decodeValue(credentialName1); + assertThat(provider, hasCredentials(TEST_ACCESS_KEY_ID, TEST_SECRET_KEY)); + } + + @Test + public void testTwoProviders() { + AWSCredentialsProvider provider = decoder.decodeValue(credentialName1 + "," + credentialName1); + assertThat(provider, hasCredentials(TEST_ACCESS_KEY_ID, TEST_SECRET_KEY)); + } + + @Test + public void testProfileProviderWithOneArg() { + AWSCredentialsProvider provider = decoder.decodeValue(credentialName2 + "|arg"); + assertThat(provider, hasCredentials("arg", "blank")); + } + + @Test + public void testProfileProviderWithTwoArgs() { + AWSCredentialsProvider provider = decoder.decodeValue(credentialName2 + "|arg1|arg2"); + assertThat(provider, hasCredentials("arg1", "arg2")); + } + + /** + * This credentials provider will always succeed + */ + public static class AlwaysSucceedCredentialsProvider implements AWSCredentialsProvider { + + @Override + public AWSCredentials getCredentials() { + return new BasicAWSCredentials(TEST_ACCESS_KEY_ID, TEST_SECRET_KEY); + } + + @Override + public void refresh() { + + } + } + + /** + * This credentials provider needs a constructor call to instantiate it + */ + public static class ConstructorCredentialsProvider implements AWSCredentialsProvider { + + private String arg1; + private String arg2; + + public ConstructorCredentialsProvider(String arg1) { + this.arg1 = arg1; + this.arg2 = "blank"; + } + + public ConstructorCredentialsProvider(String arg1, String arg2) { + this.arg1 = arg1; + this.arg2 = arg2; + } + + @Override + public AWSCredentials getCredentials() { + return new BasicAWSCredentials(arg1, arg2); + } + + @Override + public void refresh() { + + } + } +} diff --git a/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/BuilderDynaBeanTest.java b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/BuilderDynaBeanTest.java new file mode 100644 index 00000000..610230e5 --- /dev/null +++ b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/BuilderDynaBeanTest.java @@ -0,0 +1,844 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.kinesis.multilang.config; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; + +import java.util.function.Consumer; +import java.util.function.Supplier; + +import org.apache.commons.beanutils.BeanUtilsBean; +import org.apache.commons.beanutils.ConvertUtilsBean; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.RequiredArgsConstructor; +import lombok.ToString; +import lombok.experimental.Accessors; + +public class BuilderDynaBeanTest { + + private static boolean isBad = true; + private ConvertUtilsBean convertUtilsBean; + private BeanUtilsBean utilsBean; + + @Rule + public final ExpectedException thrown = ExpectedException.none(); + + @Before + public void setup() { + convertUtilsBean = new ConvertUtilsBean(); + utilsBean = new BeanUtilsBean(convertUtilsBean); + } + + @Test + public void testSimpleCreateAllParameters() throws Exception { + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleCreate.class, convertUtilsBean); + + utilsBean.setProperty(builderDynaBean, "[0]", "first"); + utilsBean.setProperty(builderDynaBean, "[1]", "last"); + + TestSimpleCreate expected = TestSimpleCreate.create("first", "last"); + TestSimpleCreate actual = builderDynaBean.build(TestSimpleCreate.class); + + assertThat(actual, equalTo(expected)); + } + + @Test + public void testSimpleCreateToManyParameters() throws Exception { + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage(containsString("exceeds the maximum")); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleCreate.class, convertUtilsBean); + + utilsBean.setProperty(builderDynaBean, "[0]", "first"); + utilsBean.setProperty(builderDynaBean, "[1]", "last"); + utilsBean.setProperty(builderDynaBean, "[2]", "age"); + + TestSimpleCreate expected = TestSimpleCreate.create("first", "last"); + TestSimpleCreate actual = builderDynaBean.build(TestSimpleCreate.class); + + assertThat(actual, equalTo(expected)); + } + + @Test + public void testSimpleCreateMissingParameter() throws Exception { + TestSimpleCreate expected = TestSimpleCreate.create(null, "last"); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleCreate.class, convertUtilsBean); + utilsBean.setProperty(builderDynaBean, "[1]", expected.lastName); + + TestSimpleCreate actual = builderDynaBean.build(TestSimpleCreate.class); + + assertThat(actual, equalTo(expected)); + } + + @Test + public void testSimpleCreateNoParameters() throws Exception { + TestSimpleCreate expected = TestSimpleCreate.create(null, null); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleCreate.class, convertUtilsBean); + utilsBean.setProperty(builderDynaBean, "[1]", expected.lastName); + + TestSimpleCreate actual = builderDynaBean.build(TestSimpleCreate.class); + + assertThat(actual, equalTo(expected)); + } + + @Test + public void testComplexCreateAllParameters() throws Exception { + TestComplexCreate expected = TestComplexCreate.create("real", + TestSimpleBuilder.builder().stringL1("l1").longVal(10L).build()); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreate.class, convertUtilsBean); + utilsBean.setProperty(builderDynaBean, "[0]", expected.realName); + utilsBean.setProperty(builderDynaBean, "[1].stringL1", expected.test1.stringL1); + utilsBean.setProperty(builderDynaBean, "[1].longVal", expected.test1.longVal); + + TestComplexCreate actual = builderDynaBean.build(TestComplexCreate.class); + + assertThat(actual, equalTo(expected)); + } + + @Test + public void testComplexCreateSimpleParameterOnly() throws Exception { + TestComplexCreate expected = TestComplexCreate.create("real", null); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreate.class, convertUtilsBean); + utilsBean.setProperty(builderDynaBean, "[0]", expected.realName); + + TestComplexCreate actual = builderDynaBean.build(TestComplexCreate.class); + + assertThat(actual, equalTo(expected)); + } + + @Test + public void testComplexCreateComplexParameterOnly() throws Exception { + TestComplexCreate expected = TestComplexCreate.create(null, + TestSimpleBuilder.builder().stringL1("l1").longVal(10L).build()); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreate.class, convertUtilsBean); + utilsBean.setProperty(builderDynaBean, "[1].stringL1", expected.test1.stringL1); + utilsBean.setProperty(builderDynaBean, "[1].longVal", expected.test1.longVal); + + TestComplexCreate actual = builderDynaBean.build(TestComplexCreate.class); + + assertThat(actual, equalTo(expected)); + } + + @Test + public void testComplexCreateNoParameters() throws Exception { + TestComplexCreate expected = TestComplexCreate.create(null, null); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreate.class, convertUtilsBean); + + TestComplexCreate actual = builderDynaBean.build(TestComplexCreate.class); + + assertThat(actual, equalTo(expected)); + } + + @Test + public void testSimpleBuilderAllParameters() throws Exception { + TestSimpleBuilder expected = TestSimpleBuilder.builder().stringL1("l1").longVal(10L).build(); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleBuilder.class, convertUtilsBean); + utilsBean.setProperty(builderDynaBean, "stringL1", expected.stringL1); + utilsBean.setProperty(builderDynaBean, "longVal", expected.longVal); + + TestSimpleBuilder actual = builderDynaBean.build(TestSimpleBuilder.class); + + assertThat(actual, equalTo(expected)); + } + + @Test + public void testSimpleBuilderMissingStringL1() throws Exception { + TestSimpleBuilder expected = TestSimpleBuilder.builder().longVal(10L).build(); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleBuilder.class, convertUtilsBean); + utilsBean.setProperty(builderDynaBean, "longVal", expected.longVal); + + TestSimpleBuilder actual = builderDynaBean.build(TestSimpleBuilder.class); + + assertThat(actual, equalTo(expected)); + } + + @Test + public void testSimpleBuilderMissingLongVal() throws Exception { + TestSimpleBuilder expected = TestSimpleBuilder.builder().stringL1("l1").build(); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleBuilder.class, convertUtilsBean); + utilsBean.setProperty(builderDynaBean, "stringL1", expected.stringL1); + + TestSimpleBuilder actual = builderDynaBean.build(TestSimpleBuilder.class); + + assertThat(actual, equalTo(expected)); + } + + @Test + public void testSimpleBuilderInvalidProperty() throws Exception { + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("Unknown property: invalidProperty"); + + TestSimpleBuilder expected = TestSimpleBuilder.builder().build(); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleBuilder.class, convertUtilsBean); + utilsBean.setProperty(builderDynaBean, "invalidProperty", "invalid"); + + TestSimpleBuilder actual = builderDynaBean.build(TestSimpleBuilder.class); + + assertThat(actual, equalTo(expected)); + } + + @Test + public void testComplexCreateSimpleBuilderVariantAllParameters() throws Exception { + TestSimpleBuilder variant = TestSimpleBuilder.builder().longVal(10L).stringL1("variant").build(); + TestComplexCreateVariance expected = TestComplexCreateVariance.create("simple-builder", variant); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean); + utilsBean.setProperty(builderDynaBean, "[0]", expected.varianceName); + utilsBean.setProperty(builderDynaBean, "[1].class", expected.variant.getClass().getName()); + utilsBean.setProperty(builderDynaBean, "[1].longVal", variant.longVal); + utilsBean.setProperty(builderDynaBean, "[1].stringL1", variant.stringL1); + + TestComplexCreateVariance actual = builderDynaBean.build(TestComplexCreateVariance.class); + + assertThat(actual, equalTo(expected)); + } + + @Test + public void testComplexCreateVariantBuilderAllParameters() throws Exception { + TestVariantBuilder variant = TestVariantBuilder.builder().variantBuilderName("variant-build").intClass(20) + .testEnum(TestEnum.Blue).build(); + TestComplexCreateVariance expected = TestComplexCreateVariance.create("builder-variant", variant); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean); + utilsBean.setProperty(builderDynaBean, "[0]", expected.varianceName); + utilsBean.setProperty(builderDynaBean, "[1].class", variant.getClass().getName()); + utilsBean.setProperty(builderDynaBean, "[1].variantBuilderName", variant.variantBuilderName); + utilsBean.setProperty(builderDynaBean, "[1].intClass", variant.intClass); + utilsBean.setProperty(builderDynaBean, "[1].testEnum", variant.testEnum); + + TestComplexCreateVariance actual = builderDynaBean.build(TestComplexCreateVariance.class); + + assertThat(actual, equalTo(expected)); + } + + @Test + public void testComplexCreateVariantCreateAllParameters() throws Exception { + TestVariantCreate variant = TestVariantCreate.create("variant-create", 100L, "varied"); + TestComplexCreateVariance expected = TestComplexCreateVariance.create("create-variant", variant); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean); + utilsBean.setProperty(builderDynaBean, "[0]", expected.varianceName); + utilsBean.setProperty(builderDynaBean, "[1].class", variant.getClass().getName()); + utilsBean.setProperty(builderDynaBean, "[1].[0]", variant.variantCreateName); + utilsBean.setProperty(builderDynaBean, "[1].[1]", variant.longClass); + utilsBean.setProperty(builderDynaBean, "[1].[2]", variant.varyString); + + TestComplexCreateVariance actual = builderDynaBean.build(TestComplexCreateVariance.class); + + assertThat(actual, equalTo(expected)); + } + + @Test + public void testComplexCreateVariantBuilderAllParametersPrefixWithJoiner() throws Exception { + TestVariantBuilder variant = TestVariantBuilder.builder().variantBuilderName("variant-build").intClass(20) + .testEnum(TestEnum.Blue).build(); + TestComplexCreateVariance expected = TestComplexCreateVariance.create("builder-variant-prefix", variant); + + String prefix = variant.getClass().getEnclosingClass().getName() + "$"; + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean, + prefix); + utilsBean.setProperty(builderDynaBean, "[0]", expected.varianceName); + utilsBean.setProperty(builderDynaBean, "[1].class", variant.getClass().getSimpleName()); + utilsBean.setProperty(builderDynaBean, "[1].variantBuilderName", variant.variantBuilderName); + utilsBean.setProperty(builderDynaBean, "[1].intClass", variant.intClass); + utilsBean.setProperty(builderDynaBean, "[1].testEnum", variant.testEnum); + + TestComplexCreateVariance actual = builderDynaBean.build(TestComplexCreateVariance.class); + + assertThat(actual, equalTo(expected)); + } + + @Test + public void testComplexCreateVariantBuilderAllParametersPrefixWithOutJoiner() throws Exception { + TestVariantBuilder variant = TestVariantBuilder.builder().variantBuilderName("variant-build").intClass(20) + .testEnum(TestEnum.Blue).build(); + TestComplexCreateVariance expected = TestComplexCreateVariance.create("builder-variant-prefix", variant); + + String prefix = variant.getClass().getEnclosingClass().getName(); + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean, + prefix); + utilsBean.setProperty(builderDynaBean, "[0]", expected.varianceName); + utilsBean.setProperty(builderDynaBean, "[1].class", variant.getClass().getSimpleName()); + utilsBean.setProperty(builderDynaBean, "[1].variantBuilderName", variant.variantBuilderName); + utilsBean.setProperty(builderDynaBean, "[1].intClass", variant.intClass); + utilsBean.setProperty(builderDynaBean, "[1].testEnum", variant.testEnum); + + TestComplexCreateVariance actual = builderDynaBean.build(TestComplexCreateVariance.class); + + assertThat(actual, equalTo(expected)); + } + + @Test + public void testComplexCreateVariantInvalidVariantClass() throws Exception { + String invalidClass = "invalid-class"; + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage(containsString("Unable to load class")); + thrown.expectMessage(containsString(invalidClass)); + thrown.expectMessage(containsString("Attempted")); + + TestComplexCreateVariance expected = TestComplexCreateVariance.create("builder-variant-prefix", null); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean); + utilsBean.setProperty(builderDynaBean, "[0]", expected.varianceName); + utilsBean.setProperty(builderDynaBean, "[1].class", invalidClass); + } + + @Test + public void testComplexCreateVariantBadLoadClass() throws Exception { + thrown.expect(ExceptionInInitializerError.class); + thrown.expectCause(instanceOf(BadClassException.class)); + TestComplexCreateVariance expected = TestComplexCreateVariance.create("builder-variant-prefix", null); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean); + utilsBean.setProperty(builderDynaBean, "[0]", expected.varianceName); + utilsBean.setProperty(builderDynaBean, "[1].class", getClass().getName() + "$BadClass"); + } + + @Test + public void testComplexRootAllParameters() throws Exception { + TestSimpleBuilder simpleBuilder = TestSimpleBuilder.builder().stringL1("simple-l1").longVal(20L).build(); + TestRootClass expected = TestRootClass.builder().intVal(10).stringVal("root").testEnum(TestEnum.Red) + .testComplexCreate(TestComplexCreate.create("real", + TestSimpleBuilder.builder().stringL1("complex-l1").longVal(10L).build())) + .testSimpleBuilder(simpleBuilder).testSimpleCreate(TestSimpleCreate.create("first", "last")).build(); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestRootClass.class, convertUtilsBean); + + utilsBean.setProperty(builderDynaBean, "intVal", expected.intVal); + utilsBean.setProperty(builderDynaBean, "stringVal", expected.stringVal); + utilsBean.setProperty(builderDynaBean, "testEnum", expected.testEnum); + utilsBean.setProperty(builderDynaBean, "testComplexCreate.[0]", expected.testComplexCreate.realName); + utilsBean.setProperty(builderDynaBean, "testComplexCreate.[1].stringL1", + expected.testComplexCreate.test1.stringL1); + utilsBean.setProperty(builderDynaBean, "testComplexCreate.[1].longVal", + expected.testComplexCreate.test1.longVal); + utilsBean.setProperty(builderDynaBean, "testSimpleBuilder.class", TestSimpleBuilder.class.getName()); + utilsBean.setProperty(builderDynaBean, "testSimpleBuilder.stringL1", simpleBuilder.stringL1); + utilsBean.setProperty(builderDynaBean, "testSimpleBuilder.longVal", simpleBuilder.longVal); + utilsBean.setProperty(builderDynaBean, "testSimpleCreate.[0]", expected.testSimpleCreate.firstName); + utilsBean.setProperty(builderDynaBean, "testSimpleCreate.[1]", expected.testSimpleCreate.lastName); + + TestRootClass actual = builderDynaBean.build(TestRootClass.class); + + assertThat(actual, equalTo(expected)); + } + + @Test + public void testComplexRootNoParameters() throws Exception { + TestRootClass expected = TestRootClass.builder().build(); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestRootClass.class, convertUtilsBean); + + TestRootClass actual = builderDynaBean.build(TestRootClass.class); + + assertThat(actual, equalTo(expected)); + } + + @Test + public void testComplexRootTopLevelOnly() throws Exception { + TestRootClass expected = TestRootClass.builder().intVal(10).stringVal("root").testEnum(TestEnum.Red).build(); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestRootClass.class, convertUtilsBean); + + utilsBean.setProperty(builderDynaBean, "intVal", expected.intVal); + utilsBean.setProperty(builderDynaBean, "stringVal", expected.stringVal); + utilsBean.setProperty(builderDynaBean, "testEnum", expected.testEnum); + + TestRootClass actual = builderDynaBean.build(TestRootClass.class); + + assertThat(actual, equalTo(expected)); + } + + @Test + public void testSupplierNotUsed() throws Exception { + TestVariantBuilder variant = TestVariantBuilder.builder().testEnum(TestEnum.Green).intClass(10) + .variantBuilderName("variant-supplier").build(); + TestSupplierClass expected = TestSupplierClass.builder().variantClass(variant).build(); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSupplierClass.class, convertUtilsBean); + utilsBean.setProperty(builderDynaBean, "variantClass.class", variant.getClass().getName()); + utilsBean.setProperty(builderDynaBean, "variantClass.testEnum", variant.testEnum); + utilsBean.setProperty(builderDynaBean, "variantClass.intClass", variant.intClass); + utilsBean.setProperty(builderDynaBean, "variantClass.variantBuilderName", variant.variantBuilderName); + + TestSupplierClass actual = builderDynaBean.build(TestSupplierClass.class); + + assertThat(actual, equalTo(expected)); + } + + @Test + public void testConsumerMethodsNotExposed() throws Exception { + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage(containsString("Unknown property: mutator")); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSupplierClass.class, convertUtilsBean); + utilsBean.setProperty(builderDynaBean, "mutator", "test-value"); + } + + @Test + public void testAttemptToBuildForWrongClass() throws Exception { + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage(containsString("cannot be assigned to")); + thrown.expectMessage(containsString(TestVariantCreate.class.getName())); + thrown.expectMessage(containsString(TestVariantBuilder.class.getName())); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestVariantBuilder.class, convertUtilsBean); + builderDynaBean.build(TestVariantCreate.class); + } + + @Test + public void testVariantBuildsToSuperType() throws Exception { + TestVariantBuilder expected = TestVariantBuilder.builder().intClass(10).testEnum(TestEnum.Green) + .variantBuilderName("variant-super").build(); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestInterface.class, convertUtilsBean); + utilsBean.setProperty(builderDynaBean, "class", expected.getClass().getName()); + utilsBean.setProperty(builderDynaBean, "intClass", expected.intClass); + utilsBean.setProperty(builderDynaBean, "testEnum", expected.testEnum); + utilsBean.setProperty(builderDynaBean, "variantBuilderName", expected.variantBuilderName); + + TestInterface actual = builderDynaBean.build(TestInterface.class); + + assertThat(actual, equalTo(expected)); + } + + @Test + public void testEmptyPropertyHandler() throws Exception { + String emptyPropertyValue = "test-property"; + TestVariantCreate expected = TestVariantCreate.create(emptyPropertyValue, (long) emptyPropertyValue.length(), + emptyPropertyValue + "-vary"); + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestInterface.class, convertUtilsBean, + s -> TestVariantCreate.create(s, (long) s.length(), s + "-vary")); + utilsBean.setProperty(builderDynaBean, "", emptyPropertyValue); + + TestInterface actual = builderDynaBean.build(TestInterface.class); + + assertThat(actual, equalTo(expected)); + } + + @Test + public void testEmptyPropertyHandlerThrowsAfterUse() throws Exception { + thrown.expect(IllegalStateException.class); + thrown.expectMessage(containsString("When a property handler is resolved further properties may not be set.")); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestInterface.class, convertUtilsBean, + s -> TestVariantCreate.create("test", 10, "test")); + utilsBean.setProperty(builderDynaBean, "", "test"); + utilsBean.setProperty(builderDynaBean, "[0]", "test"); + } + + @Test + public void testEmptyPropertyReturnsInvalidObject() throws Exception { + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage(containsString(TestEnum.class.getName())); + thrown.expectMessage(containsString(TestInterface.class.getName())); + thrown.expectMessage(containsString("cannot be assigned to")); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestInterface.class, convertUtilsBean, + s -> TestEnum.Green); + + utilsBean.setProperty(builderDynaBean, "", "test"); + + builderDynaBean.build(TestInterface.class); + } + + @Test + public void testSimpleArrayValues() throws Exception { + SimpleArrayClassVariant expected = SimpleArrayClassVariant.builder().ints(new Integer[] { 1, 2, 3 }) + .variantName("simple-array").longs(new Long[] { 1L, 2L, 3L }).strings(new String[] { "a", "b", "c" }) + .build(); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(SimpleArrayClassVariant.class, convertUtilsBean); + utilsBean.setProperty(builderDynaBean, "variantName", expected.variantName); + for (int i = 0; i < expected.strings.length; ++i) { + utilsBean.setProperty(builderDynaBean, "strings[" + i + "]", expected.strings[i]); + } + + for (int i = 0; i < expected.ints.length; ++i) { + utilsBean.setProperty(builderDynaBean, "ints[" + i + "]", expected.ints[i]); + } + + for (int i = 0; i < expected.longs.length; ++i) { + utilsBean.setProperty(builderDynaBean, "longs[" + i + "]", expected.longs[i]); + } + + SimpleArrayClassVariant actual = builderDynaBean.build(SimpleArrayClassVariant.class); + + assertThat(actual, equalTo(expected)); + } + + @Test + public void testComplexArrayValuesBuilder() throws Exception { + TestVariantBuilder variant1 = TestVariantBuilder.builder().variantBuilderName("variant-1") + .testEnum(TestEnum.Green).intClass(10).build(); + TestVariantBuilder variant2 = TestVariantBuilder.builder().variantBuilderName("variant-2") + .testEnum(TestEnum.Blue).intClass(20).build(); + ComplexArrayClassVariant expected = ComplexArrayClassVariant.builder().variantName("complex-test") + .tests(new TestInterface[] { variant1, variant2 }).build(); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(ComplexArrayClassVariant.class, convertUtilsBean); + + utilsBean.setProperty(builderDynaBean, "variantName", expected.variantName); + utilsBean.setProperty(builderDynaBean, "tests[0].class", TestVariantBuilder.class.getName()); + utilsBean.setProperty(builderDynaBean, "tests[0].variantBuilderName", variant1.variantBuilderName); + utilsBean.setProperty(builderDynaBean, "tests[0].intClass", variant1.intClass); + utilsBean.setProperty(builderDynaBean, "tests[0].testEnum", variant1.testEnum); + + utilsBean.setProperty(builderDynaBean, "tests[1].class", TestVariantBuilder.class.getName()); + utilsBean.setProperty(builderDynaBean, "tests[1].variantBuilderName", variant2.variantBuilderName); + utilsBean.setProperty(builderDynaBean, "tests[1].intClass", variant2.intClass); + utilsBean.setProperty(builderDynaBean, "tests[1].testEnum", variant2.testEnum); + + ComplexArrayClassVariant actual = builderDynaBean.build(ComplexArrayClassVariant.class); + + assertThat(actual, equalTo(expected)); + } + + @Test + public void testComplexArrayValuesCreate() throws Exception { + TestVariantCreate variant1 = TestVariantCreate.create("variant-1", 10L, "vary-1"); + TestVariantCreate variant2 = TestVariantCreate.create("variant-2", 20L, "vary-2"); + + ComplexArrayClassVariant expected = ComplexArrayClassVariant.builder().variantName("create-test") + .tests(new TestInterface[] { variant1, variant2 }).build(); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(ComplexArrayClassVariant.class, convertUtilsBean); + + utilsBean.setProperty(builderDynaBean, "variantName", expected.variantName); + utilsBean.setProperty(builderDynaBean, "tests[0].class", variant1.getClass().getName()); + utilsBean.setProperty(builderDynaBean, "tests[0].[0]", variant1.variantCreateName); + utilsBean.setProperty(builderDynaBean, "tests[0].[1]", variant1.longClass); + utilsBean.setProperty(builderDynaBean, "tests[0].[2]", variant1.varyString); + + utilsBean.setProperty(builderDynaBean, "tests[1].class", variant2.getClass().getName()); + utilsBean.setProperty(builderDynaBean, "tests[1].[0]", variant2.variantCreateName); + utilsBean.setProperty(builderDynaBean, "tests[1].[1]", variant2.longClass); + utilsBean.setProperty(builderDynaBean, "tests[1].[2]", variant2.varyString); + + ComplexArrayClassVariant actual = builderDynaBean.build(ComplexArrayClassVariant.class); + + assertThat(actual, equalTo(expected)); + + } + + @Test + public void testComplexArrayValuesMixed() throws Exception { + TestInterface[] variants = new TestInterface[10]; + for (int i = 0; i < variants.length; ++i) { + if (i % 2 == 0) { + variants[i] = TestVariantCreate.create("create-variant-" + i, i + 5, "vary-" + i); + } else { + variants[i] = TestVariantBuilder.builder().testEnum(TestEnum.values()[i % TestEnum.values().length]) + .intClass(i).variantBuilderName("builder-variant-" + i).build(); + } + } + + ComplexArrayClassVariant expected = ComplexArrayClassVariant.builder().variantName("large-complex") + .tests(variants).build(); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(ComplexArrayClassVariant.class, convertUtilsBean); + + utilsBean.setProperty(builderDynaBean, "variantName", expected.variantName); + for (int i = 0; i < variants.length; ++i) { + String prefix = "tests[" + i + "]."; + TestInterface variant = variants[i]; + if (variant instanceof TestVariantCreate) { + TestVariantCreate create = (TestVariantCreate) variant; + utilsBean.setProperty(builderDynaBean, prefix + "class", create.getClass().getName()); + utilsBean.setProperty(builderDynaBean, prefix + "[0]", create.variantCreateName); + utilsBean.setProperty(builderDynaBean, prefix + "[1]", create.longClass); + utilsBean.setProperty(builderDynaBean, prefix + "[2]", create.varyString); + } else if (variant instanceof TestVariantBuilder) { + TestVariantBuilder builder = (TestVariantBuilder) variant; + utilsBean.setProperty(builderDynaBean, prefix + "class", builder.getClass().getName()); + utilsBean.setProperty(builderDynaBean, prefix + "variantBuilderName", builder.variantBuilderName); + utilsBean.setProperty(builderDynaBean, prefix + "intClass", builder.intClass); + utilsBean.setProperty(builderDynaBean, prefix + "testEnum", builder.testEnum); + } else { + fail("Unknown variant " + variants[i].getClass().getName()); + } + } + + ComplexArrayClassVariant actual = builderDynaBean.build(ComplexArrayClassVariant.class); + + assertThat(actual, equalTo(expected)); + } + + @Test + public void testInvalidBuilderCreateClassBuild() throws Exception { + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestInterface.class, convertUtilsBean); + + TestInterface actual = builderDynaBean.build(TestInterface.class); + + assertThat(actual, nullValue()); + } + + @Test + public void testInvalidBuilderCreateClassSetProperty() throws Exception { + thrown.expect(IllegalStateException.class); + thrown.expectMessage(containsString("Unable to to introspect or handle")); + thrown.expectMessage(containsString(TestInterface.class.getName())); + thrown.expectMessage(containsString("as it doesn't have a builder or create method")); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestInterface.class, convertUtilsBean); + + utilsBean.setProperty(builderDynaBean, "testProperty", "test"); + } + + @Test + public void testSetMapAccessThrowsException() throws Exception { + thrown.expect(UnsupportedOperationException.class); + thrown.expectMessage(BuilderDynaBean.NO_MAP_ACCESS_SUPPORT); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleBuilder.class, convertUtilsBean); + + utilsBean.setProperty(builderDynaBean, "stringL1(value)", "test"); + } + + @Test + public void testGetMapAccessThrowsException() throws Exception { + thrown.expect(UnsupportedOperationException.class); + thrown.expectMessage(BuilderDynaBean.NO_MAP_ACCESS_SUPPORT); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleBuilder.class, convertUtilsBean); + // + // We directly access the get method as there is no way to trigger utilsBean to access it + // + builderDynaBean.get("stringL1", "value"); + } + + @Test + public void testRemoveThrowsException() throws Exception { + thrown.expect(UnsupportedOperationException.class); + thrown.expectMessage(BuilderDynaBean.NO_MAP_ACCESS_SUPPORT); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleBuilder.class, convertUtilsBean); + // + // We directly access the remove method as there is no way to trigger utilsBean to access it + // + builderDynaBean.remove("stringL1", "value"); + } + + @Test + public void testContainsThrowsException() throws Exception { + thrown.expect(UnsupportedOperationException.class); + thrown.expectMessage(BuilderDynaBean.NO_MAP_ACCESS_SUPPORT); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleBuilder.class, convertUtilsBean); + // + // We directly access the remove method as there is no way to trigger utilsBean to access it + // + builderDynaBean.contains("stringL1", "value"); + } + + @Test + public void testAdditionalMutators() throws Exception { + TestSimpleBuilder expected = TestSimpleBuilder.builder().stringL1("test").longVal(10L).build(); + + BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleBuilder.class, convertUtilsBean); + + utilsBean.setProperty(builderDynaBean, "stringL1", expected.stringL1); + + TestSimpleBuilder actual = builderDynaBean.build(TestSimpleBuilder.class, + b -> ((TestSimpleBuilder.TestSimpleBuilderBuilder) b).longVal(expected.longVal)); + + assertThat(actual, equalTo(expected)); + } + + public enum TestEnum { + Red, Green, Blue + } + + public interface TestInterface { + + } + + @Accessors(fluent = true) + @ToString + @EqualsAndHashCode + @RequiredArgsConstructor + public static class TestSimpleCreate { + private final String firstName; + private final String lastName; + + public static TestSimpleCreate create(String firstName, String lastName) { + return new TestSimpleCreate(firstName, lastName); + } + } + + @Accessors(fluent = true) + @ToString + @EqualsAndHashCode + @RequiredArgsConstructor + public static class TestComplexCreate { + private final String realName; + private final TestSimpleBuilder test1; + + public static TestComplexCreate create(String realName, TestSimpleBuilder test1) { + return new TestComplexCreate(realName, test1); + } + } + + @Accessors(fluent = true) + @ToString + @EqualsAndHashCode + @RequiredArgsConstructor + public static class TestComplexCreateVariance { + private final String varianceName; + private final TestInterface variant; + + public static TestComplexCreateVariance create(String varianceName, TestInterface variant) { + return new TestComplexCreateVariance(varianceName, variant); + } + } + + @Builder + @Accessors(fluent = true) + @ToString + @EqualsAndHashCode + public static class TestSimpleBuilder implements TestInterface { + private String stringL1; + private long longVal; + } + + @Builder + @Accessors(fluent = true) + @ToString + @EqualsAndHashCode + public static class TestVariantBuilder implements TestInterface { + private String variantBuilderName; + private TestEnum testEnum; + private Integer intClass; + } + + @Accessors(fluent = true) + @ToString + @EqualsAndHashCode + @RequiredArgsConstructor + public static class TestVariantCreate implements TestInterface { + private final String variantCreateName; + private final long longClass; + private final String varyString; + + public static TestVariantCreate create(String variantCreateName, long longClass, String varyString) { + return new TestVariantCreate(variantCreateName, longClass, varyString); + } + } + + @Builder + @Accessors(fluent = true) + @ToString + @EqualsAndHashCode + public static class TestRootClass { + private String stringVal; + private int intVal; + private TestEnum testEnum; + TestSimpleCreate testSimpleCreate; + TestComplexCreate testComplexCreate; + TestSimpleBuilder testSimpleBuilder; + } + + @ToString + @EqualsAndHashCode + public static class TestSupplierClass { + private TestInterface variantClass; + + public static TestSupplierClassBuilder builder() { + return new TestSupplierClassBuilder(); + } + } + + @Builder + @Accessors(fluent = true) + @ToString + @EqualsAndHashCode + public static class SimpleArrayClassVariant implements TestInterface { + private String variantName; + private String[] strings; + private Integer[] ints; + private Long[] longs; + } + + @Builder + @Accessors(fluent = true) + @ToString + @EqualsAndHashCode + public static class ComplexArrayClassVariant implements TestInterface { + private String variantName; + private TestInterface[] tests; + } + + public static class TestSupplierClassBuilder { + private TestSupplierClass testSupplierClass = new TestSupplierClass(); + + public TestSupplierClassBuilder variantClass(TestInterface testInterface) { + testSupplierClass.variantClass = testInterface; + return this; + } + + public TestSupplierClassBuilder variantClass(Supplier supplier) { + throw new IllegalStateException("Supplier method should not be used."); + } + + public TestSupplierClassBuilder mutator(Consumer consumer) { + consumer.accept(this); + return this; + } + + public TestSupplierClass build() { + return testSupplierClass; + } + } + + public static class BadClassException extends RuntimeException { + public BadClassException(String message) { + super(message); + } + } + + public static class BadClass { + static { + if (BuilderDynaBeanTest.isBad) { + throw new BadClassException("This is a bad class"); + } + } + + public String name = "default"; + + } + +} \ No newline at end of file diff --git a/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/ConfigurationSettableUtilsTest.java b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/ConfigurationSettableUtilsTest.java new file mode 100644 index 00000000..d864cdc4 --- /dev/null +++ b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/ConfigurationSettableUtilsTest.java @@ -0,0 +1,179 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.kinesis.multilang.config; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.junit.Assert.assertThat; + +import java.util.Optional; + +import org.junit.Test; + +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.Accessors; + +public class ConfigurationSettableUtilsTest { + + @Test + public void testNoPropertiesSet() { + ConfigResult expected = ConfigResult.builder().build(); + + ConfigObject configObject = ConfigObject.builder().build(); + ConfigResult actual = resolve(configObject); + + assertThat(actual, equalTo(expected)); + } + + @Test + public void testPrimitivesSet() { + ConfigResult expected = ConfigResult.builder().rawInt(10).rawLong(15L).build(); + + ConfigObject configObject = ConfigObject.builder().rawInt(expected.rawInt).rawLong(expected.rawLong).build(); + ConfigResult actual = resolve(configObject); + + assertThat(actual, equalTo(expected)); + } + + @Test + public void testHeapValuesSet() { + ConfigResult expected = ConfigResult.builder().name("test").boxedInt(10).boxedLong(15L).build(); + + ConfigObject configObject = ConfigObject.builder().name(expected.name).boxedInt(expected.boxedInt.intValue()) + .boxedLong(expected.boxedLong.longValue()).build(); + ConfigResult actual = resolve(configObject); + + assertThat(actual, equalTo(expected)); + } + + @Test + public void testComplexValuesSet() { + ComplexValue complexValue = ComplexValue.builder().name("complex").value(10).build(); + ConfigResult expected = ConfigResult.builder().complexValue(complexValue).build(); + + ConfigObject configObject = ConfigObject.builder() + .complexValue(ComplexValue.builder().name(complexValue.name).value(complexValue.value).build()).build(); + ConfigResult actual = resolve(configObject); + + assertThat(actual, equalTo(expected)); + + } + + @Test + public void testOptionalValuesSet() { + ComplexValue complexValue = ComplexValue.builder().name("optional-complex").value(20).build(); + ConfigResult expected = ConfigResult.builder().optionalString(Optional.of("test")) + .optionalInteger(Optional.of(10)).optionalLong(Optional.of(15L)) + .optionalComplexValue(Optional.of(complexValue)).build(); + + ConfigObject configObject = ConfigObject.builder().optionalString(expected.optionalString.get()) + .optionalInteger(expected.optionalInteger.get()).optionalLong(expected.optionalLong.get()) + .optionalComplexValue(expected.optionalComplexValue.get()).build(); + ConfigResult actual = resolve(configObject); + + assertThat(actual, equalTo(expected)); + } + + @Test + public void testRenamedRawValues() { + ComplexValue complexValue = ComplexValue.builder().name("renamed-complex").value(20).build(); + ConfigResult expected = ConfigResult.builder().renamedString("renamed").renamedInt(10) + .renamedOptionalString(Optional.of("renamed-optional")).renamedComplexValue(complexValue).build(); + + ConfigObject configObject = ConfigObject.builder().toRenameString(expected.renamedString) + .toRenameInt(expected.renamedInt).toRenameComplexValue(complexValue) + .optionalToRename(expected.renamedOptionalString.get()).build(); + ConfigResult actual = resolve(configObject); + + assertThat(actual, equalTo(expected)); + } + + private ConfigResult resolve(ConfigObject configObject) { + return ConfigurationSettableUtils.resolveFields(configObject, ConfigResult.builder().build()); + } + + @Accessors(fluent = true) + @Builder + @Getter + @Setter + @EqualsAndHashCode + public static class ConfigResult { + private String name; + private int rawInt; + private Integer boxedInt; + private long rawLong; + private Long boxedLong; + private ComplexValue complexValue; + + private Optional optionalString; + private Optional optionalInteger; + private Optional optionalLong; + private Optional optionalComplexValue; + + private String renamedString; + private int renamedInt; + private Optional renamedOptionalString; + private ComplexValue renamedComplexValue; + + } + + @Accessors(fluent = true) + @Builder + @EqualsAndHashCode + public static class ComplexValue { + private String name; + private int value; + } + + @Builder + public static class ConfigObject { + + @ConfigurationSettable(configurationClass = ConfigResult.class) + private String name; + @ConfigurationSettable(configurationClass = ConfigResult.class) + private int rawInt; + @ConfigurationSettable(configurationClass = ConfigResult.class) + private Integer boxedInt; + @ConfigurationSettable(configurationClass = ConfigResult.class) + private long rawLong; + @ConfigurationSettable(configurationClass = ConfigResult.class) + private Long boxedLong; + @ConfigurationSettable(configurationClass = ConfigResult.class) + private ComplexValue complexValue; + + @ConfigurationSettable(configurationClass = ConfigResult.class, convertToOptional = true) + private String optionalString; + @ConfigurationSettable(configurationClass = ConfigResult.class, convertToOptional = true) + private Integer optionalInteger; + @ConfigurationSettable(configurationClass = ConfigResult.class, convertToOptional = true) + private Long optionalLong; + @ConfigurationSettable(configurationClass = ConfigResult.class, convertToOptional = true) + private ComplexValue optionalComplexValue; + + @ConfigurationSettable(configurationClass = ConfigResult.class, methodName = "renamedString") + private String toRenameString; + @ConfigurationSettable(configurationClass = ConfigResult.class, methodName = "renamedInt") + private int toRenameInt; + @ConfigurationSettable(configurationClass = ConfigResult.class, methodName = "renamedOptionalString", convertToOptional = true) + private String optionalToRename; + @ConfigurationSettable(configurationClass = ConfigResult.class, methodName = "renamedComplexValue") + private ComplexValue toRenameComplexValue; + + } + +} \ No newline at end of file diff --git a/amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/config/DatePropertyValueDecoderTest.java b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/DatePropertyValueDecoderTest.java similarity index 96% rename from amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/config/DatePropertyValueDecoderTest.java rename to amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/DatePropertyValueDecoderTest.java index 7c5cd8f3..2d417b19 100644 --- a/amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/config/DatePropertyValueDecoderTest.java +++ b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/DatePropertyValueDecoderTest.java @@ -12,7 +12,7 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package com.amazonaws.services.kinesis.multilang.config; +package software.amazon.kinesis.multilang.config; import static org.junit.Assert.assertEquals; diff --git a/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/FanoutConfigBeanTest.java b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/FanoutConfigBeanTest.java new file mode 100644 index 00000000..fd9e88de --- /dev/null +++ b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/FanoutConfigBeanTest.java @@ -0,0 +1,67 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.kinesis.multilang.config; + +import org.apache.commons.beanutils.BeanUtilsBean; +import org.apache.commons.beanutils.ConvertUtilsBean; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; + +import software.amazon.awssdk.services.kinesis.KinesisAsyncClient; +import software.amazon.kinesis.retrieval.fanout.FanOutConfig; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.junit.Assert.assertThat; + +@RunWith(MockitoJUnitRunner.class) +public class FanoutConfigBeanTest { + + @Mock + private KinesisAsyncClient kinesisAsyncClient; + + @Test + public void testAllConfigurationTransits() { + FanoutConfigBean fanoutConfigBean = new FanoutConfigBean(); + fanoutConfigBean.setConsumerArn("consumer-arn"); + fanoutConfigBean.setConsumerName("consumer-name"); + fanoutConfigBean.setMaxDescribeStreamConsumerRetries(10); + fanoutConfigBean.setMaxDescribeStreamSummaryRetries(20); + fanoutConfigBean.setRegisterStreamConsumerRetries(30); + fanoutConfigBean.setRetryBackoffMillis(1000); + + ConvertUtilsBean convertUtilsBean = new ConvertUtilsBean(); + BeanUtilsBean utilsBean = new BeanUtilsBean(convertUtilsBean); + + MultiLangDaemonConfiguration configuration = new MultiLangDaemonConfiguration(utilsBean, convertUtilsBean); + configuration.setStreamName("test-stream"); + configuration.setApplicationName("test-application"); + FanOutConfig fanOutConfig =fanoutConfigBean.build(kinesisAsyncClient, configuration); + + assertThat(fanOutConfig.kinesisClient(), equalTo(kinesisAsyncClient)); + assertThat(fanOutConfig.streamName(), equalTo(configuration.getStreamName())); + assertThat(fanOutConfig.applicationName(), equalTo(configuration.getApplicationName())); + assertThat(fanOutConfig.consumerArn(), equalTo(fanoutConfigBean.getConsumerArn())); + assertThat(fanOutConfig.consumerName(), equalTo(fanoutConfigBean.getConsumerName())); + assertThat(fanOutConfig.maxDescribeStreamConsumerRetries(), equalTo(fanoutConfigBean.getMaxDescribeStreamConsumerRetries())); + assertThat(fanOutConfig.maxDescribeStreamSummaryRetries(), equalTo(fanoutConfigBean.getMaxDescribeStreamSummaryRetries())); + assertThat(fanOutConfig.registerStreamConsumerRetries(), equalTo(fanoutConfigBean.getRegisterStreamConsumerRetries())); + assertThat(fanOutConfig.retryBackoffMillis(), equalTo(fanoutConfigBean.getRetryBackoffMillis())); + + } + +} \ No newline at end of file diff --git a/amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/config/KinesisClientLibConfiguratorTest.java b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/KinesisClientLibConfiguratorTest.java similarity index 68% rename from amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/config/KinesisClientLibConfiguratorTest.java rename to amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/KinesisClientLibConfiguratorTest.java index 687d8f40..8e0baa86 100644 --- a/amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/config/KinesisClientLibConfiguratorTest.java +++ b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/KinesisClientLibConfiguratorTest.java @@ -12,55 +12,77 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package com.amazonaws.services.kinesis.multilang.config; +package software.amazon.kinesis.multilang.config; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.nullValue; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.ByteArrayInputStream; import java.io.InputStream; +import java.net.URI; +import java.util.Arrays; +import java.util.HashSet; import java.util.Optional; import java.util.Set; +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; import org.apache.commons.lang3.StringUtils; import org.junit.Ignore; +import org.junit.Rule; import org.junit.Test; import com.google.common.collect.ImmutableSet; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.kinesis.common.InitialPositionInStream; import software.amazon.kinesis.coordinator.KinesisClientLibConfiguration; import software.amazon.kinesis.metrics.MetricsLevel; +import software.amazon.kinesis.processor.ShardRecordProcessorFactory; +@RunWith(MockitoJUnitRunner.class) public class KinesisClientLibConfiguratorTest { - private String credentialName1 = "com.amazonaws.services.kinesis.multilang.config.KinesisClientLibConfiguratorTest$AlwaysSucceedCredentialsProvider"; - private String credentialName2 = "com.amazonaws.services.kinesis.multilang.config.KinesisClientLibConfiguratorTest$AlwaysFailCredentialsProvider"; - private String credentialNameKinesis = "com.amazonaws.services.kinesis.multilang.config.KinesisClientLibConfiguratorTest$AlwaysSucceedCredentialsProviderKinesis"; - private String credentialNameDynamoDB = "com.amazonaws.services.kinesis.multilang.config.KinesisClientLibConfiguratorTest$AlwaysSucceedCredentialsProviderDynamoDB"; - private String credentialNameCloudWatch = "com.amazonaws.services.kinesis.multilang.config.KinesisClientLibConfiguratorTest$AlwaysSucceedCredentialsProviderCloudWatch"; + private String credentialName1 = "software.amazon.kinesis.multilang.config.KinesisClientLibConfiguratorTest$AlwaysSucceedCredentialsProvider"; + private String credentialName2 = "software.amazon.kinesis.multilang.config.KinesisClientLibConfiguratorTest$AlwaysFailCredentialsProvider"; + private String credentialNameKinesis = "software.amazon.kinesis.multilang.config.KinesisClientLibConfiguratorTest$AlwaysSucceedCredentialsProviderKinesis"; + private String credentialNameDynamoDB = "software.amazon.kinesis.multilang.config.KinesisClientLibConfiguratorTest$AlwaysSucceedCredentialsProviderDynamoDB"; + private String credentialNameCloudWatch = "software.amazon.kinesis.multilang.config.KinesisClientLibConfiguratorTest$AlwaysSucceedCredentialsProviderCloudWatch"; private KinesisClientLibConfigurator configurator = new KinesisClientLibConfigurator(); + @Rule + public final ExpectedException thrown = ExpectedException.none(); + + @Mock + private ShardRecordProcessorFactory shardRecordProcessorFactory; + @Test public void testWithBasicSetup() { - KinesisClientLibConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a", + MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a", "applicationName = b", "AWSCredentialsProvider = " + credentialName1, "workerId = 123" }, '\n')); assertEquals(config.getApplicationName(), "b"); assertEquals(config.getStreamName(), "a"); assertEquals(config.getWorkerIdentifier(), "123"); - assertEquals(config.getMaxGetRecordsThreadPool(), Optional.empty()); - assertEquals(config.getRetryGetRecordsInSeconds(), Optional.empty()); + assertThat(config.getMaxGetRecordsThreadPool(), nullValue()); + assertThat(config.getRetryGetRecordsInSeconds(), nullValue()); } @Test public void testWithLongVariables() { - KinesisClientLibConfiguration config = getConfiguration(StringUtils.join(new String[] { "applicationName = app", + MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "applicationName = app", "streamName = 123", "AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2, "workerId = 123", "failoverTimeMillis = 100", "shardSyncIntervalMillis = 500" }, '\n')); @@ -73,7 +95,7 @@ public class KinesisClientLibConfiguratorTest { @Test public void testWithUnsupportedClientConfigurationVariables() { - KinesisClientLibConfiguration config = getConfiguration(StringUtils.join( + MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join( new String[] { "AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2, "workerId = id", "kinesisClientConfig = {}", "streamName = stream", "applicationName = b" }, '\n')); @@ -86,7 +108,7 @@ public class KinesisClientLibConfiguratorTest { @Test public void testWithIntVariables() { - KinesisClientLibConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = kinesis", + MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = kinesis", "AWSCredentialsProvider = " + credentialName2 + ", " + credentialName1, "workerId = w123", "maxRecords = 10", "metricsMaxQueueSize = 20", "applicationName = kinesis", "retryGetRecordsInSeconds = 2", "maxGetRecordsThreadPool = 1" }, '\n')); @@ -96,13 +118,13 @@ public class KinesisClientLibConfiguratorTest { assertEquals(config.getWorkerIdentifier(), "w123"); assertEquals(config.getMaxRecords(), 10); assertEquals(config.getMetricsMaxQueueSize(), 20); - assertEquals(config.getRetryGetRecordsInSeconds(), Optional.of(2)); - assertEquals(config.getMaxGetRecordsThreadPool(), Optional.of(1)); + assertThat(config.getRetryGetRecordsInSeconds(), equalTo(2)); + assertThat(config.getMaxGetRecordsThreadPool(), equalTo(1)); } @Test public void testWithBooleanVariables() { - KinesisClientLibConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a", + MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a", "applicationName = b", "AWSCredentialsProvider = ABCD, " + credentialName1, "workerId = 0", "cleanupLeasesUponShardCompletion = false", "validateSequenceNumberBeforeCheckpointing = true" }, '\n')); @@ -110,36 +132,35 @@ public class KinesisClientLibConfiguratorTest { assertEquals(config.getApplicationName(), "b"); assertEquals(config.getStreamName(), "a"); assertEquals(config.getWorkerIdentifier(), "0"); - assertFalse(config.shouldCleanupLeasesUponShardCompletion()); - assertTrue(config.shouldValidateSequenceNumberBeforeCheckpointing()); + assertFalse(config.isCleanupLeasesUponShardCompletion()); + assertTrue(config.isValidateSequenceNumberBeforeCheckpointing()); } @Test public void testWithStringVariables() { - KinesisClientLibConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a", + MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a", "applicationName = b", "AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 1", "kinesisEndpoint = https://kinesis", "metricsLevel = SUMMARY" }, '\n')); assertEquals(config.getWorkerIdentifier(), "1"); - assertEquals(config.getKinesisEndpoint(), "https://kinesis"); + assertEquals(config.getKinesisClient().get("endpointOverride"), URI.create("https://kinesis")); assertEquals(config.getMetricsLevel(), MetricsLevel.SUMMARY); } @Test public void testWithSetVariables() { - KinesisClientLibConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a", + MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a", "applicationName = b", "AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 1", "metricsEnabledDimensions = ShardId, WorkerIdentifier" }, '\n')); Set expectedMetricsEnabledDimensions = ImmutableSet. builder() - .add("ShardId", "WorkerIdentifier") - .addAll(KinesisClientLibConfiguration.METRICS_ALWAYS_ENABLED_DIMENSIONS).build(); - assertEquals(config.getMetricsEnabledDimensions(), expectedMetricsEnabledDimensions); + .add("ShardId", "WorkerIdentifier").build(); + assertThat(new HashSet<>(Arrays.asList(config.getMetricsEnabledDimensions())), equalTo(expectedMetricsEnabledDimensions)); } @Test public void testWithInitialPositionInStreamVariables() { - KinesisClientLibConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a", + MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a", "applicationName = b", "AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 123", "initialPositionInStream = TriM_Horizon" }, '\n')); @@ -148,7 +169,7 @@ public class KinesisClientLibConfiguratorTest { @Test public void testSkippingNonKCLVariables() { - KinesisClientLibConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a", + MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a", "applicationName = b", "AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 123", "initialPositionInStream = TriM_Horizon", "abc = 1" }, '\n')); @@ -160,11 +181,11 @@ public class KinesisClientLibConfiguratorTest { @Test public void testEmptyOptionalVariables() { - KinesisClientLibConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a", + MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a", "applicationName = b", "AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 123", "initialPositionInStream = TriM_Horizon", "maxGetRecordsThreadPool = 1" }, '\n')); - assertEquals(config.getMaxGetRecordsThreadPool(), Optional.of(1)); - assertEquals(config.getRetryGetRecordsInSeconds(), Optional.empty()); + assertThat(config.getMaxGetRecordsThreadPool(), equalTo(1)); + assertThat(config.getRetryGetRecordsInSeconds(), nullValue()); } @Test @@ -212,17 +233,15 @@ public class KinesisClientLibConfiguratorTest { @Test public void testWithMissingCredentialsProvider() { + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("A basic set of AWS credentials must be provided"); + String test = StringUtils.join(new String[] { "streamName = a", "applicationName = b", "workerId = 123", "failoverTimeMillis = 100", "shardSyncIntervalMillis = 500" }, '\n'); InputStream input = new ByteArrayInputStream(test.getBytes()); // separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement - try { - configurator.getConfiguration(input); - fail("expect failure with no credentials provider variables"); - } catch (Exception e) { - // succeed - } + configurator.getConfiguration(input); } @Test @@ -232,7 +251,7 @@ public class KinesisClientLibConfiguratorTest { "failoverTimeMillis = 100", "shardSyncIntervalMillis = 500" }, '\n'); InputStream input = new ByteArrayInputStream(test.getBytes()); - KinesisClientLibConfiguration config = configurator.getConfiguration(input); + MultiLangDaemonConfiguration config = configurator.getConfiguration(input); // if workerId is not provided, configurator should assign one for it automatically assertNotNull(config.getWorkerIdentifier()); @@ -241,32 +260,25 @@ public class KinesisClientLibConfiguratorTest { @Test public void testWithMissingStreamName() { + thrown.expect(NullPointerException.class); + thrown.expectMessage("Stream name is required"); + String test = StringUtils.join(new String[] { "applicationName = b", "AWSCredentialsProvider = " + credentialName1, "workerId = 123", "failoverTimeMillis = 100" }, '\n'); InputStream input = new ByteArrayInputStream(test.getBytes()); - // separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement - try { - configurator.getConfiguration(input); - fail("expect failure with no stream name variables"); - } catch (Exception e) { - // succeed - } + configurator.getConfiguration(input); } @Test public void testWithMissingApplicationName() { + thrown.expect(NullPointerException.class); + thrown.expectMessage("Application name is required"); + String test = StringUtils.join(new String[] { "streamName = a", "AWSCredentialsProvider = " + credentialName1, "workerId = 123", "failoverTimeMillis = 100" }, '\n'); InputStream input = new ByteArrayInputStream(test.getBytes()); - - // separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement - try { - configurator.getConfiguration(input); - fail("expect failure with no application variables"); - } catch (Exception e) { - // succeed - } + configurator.getConfiguration(input); } @Test @@ -279,8 +291,8 @@ public class KinesisClientLibConfiguratorTest { // separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement try { - KinesisClientLibConfiguration config = configurator.getConfiguration(input); - config.getKinesisCredentialsProvider().resolveCredentials(); + MultiLangDaemonConfiguration config = configurator.getConfiguration(input); + config.getKinesisCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials(); fail("expect failure with wrong credentials provider"); } catch (Exception e) { // succeed @@ -289,7 +301,6 @@ public class KinesisClientLibConfiguratorTest { // TODO: fix this test @Test - @Ignore public void testWithDifferentAWSCredentialsForDynamoDBAndCloudWatch() { String test = StringUtils.join(new String[] { "streamName = a", "applicationName = b", "AWSCredentialsProvider = " + credentialNameKinesis, @@ -299,19 +310,19 @@ public class KinesisClientLibConfiguratorTest { InputStream input = new ByteArrayInputStream(test.getBytes()); // separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement - KinesisClientLibConfiguration config = configurator.getConfiguration(input); + MultiLangDaemonConfiguration config = configurator.getConfiguration(input); try { - config.getKinesisCredentialsProvider().resolveCredentials(); + config.getKinesisCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials(); } catch (Exception e) { fail("Kinesis credential providers should not fail."); } try { - config.getDynamoDBCredentialsProvider().resolveCredentials(); + config.getDynamoDBCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials(); } catch (Exception e) { fail("DynamoDB credential providers should not fail."); } try { - config.getCloudWatchCredentialsProvider().resolveCredentials(); + config.getCloudWatchCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials(); } catch (Exception e) { fail("CloudWatch credential providers should not fail."); } @@ -319,32 +330,31 @@ public class KinesisClientLibConfiguratorTest { // TODO: fix this test @Test - @Ignore public void testWithDifferentAWSCredentialsForDynamoDBAndCloudWatchFailed() { String test = StringUtils.join(new String[] { "streamName = a", "applicationName = b", "AWSCredentialsProvider = " + credentialNameKinesis, - "AWSCredentialsProviderDynamoDB = " + credentialName1, - "AWSCredentialsProviderCloudWatch = " + credentialName1, "failoverTimeMillis = 100", + "AWSCredentialsProviderDynamoDB = " + credentialName2, + "AWSCredentialsProviderCloudWatch = " + credentialName2, "failoverTimeMillis = 100", "shardSyncIntervalMillis = 500" }, '\n'); InputStream input = new ByteArrayInputStream(test.getBytes()); // separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement // separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement - KinesisClientLibConfiguration config = configurator.getConfiguration(input); + MultiLangDaemonConfiguration config = configurator.getConfiguration(input); try { - config.getKinesisCredentialsProvider().resolveCredentials(); + config.getKinesisCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials(); } catch (Exception e) { fail("Kinesis credential providers should not fail."); } try { - config.getDynamoDBCredentialsProvider().resolveCredentials(); + config.getDynamoDBCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials(); fail("DynamoDB credential providers should fail."); } catch (Exception e) { // succeed } try { - config.getCloudWatchCredentialsProvider().resolveCredentials(); + config.getCloudWatchCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials(); fail("CloudWatch credential providers should fail."); } catch (Exception e) { // succeed @@ -354,66 +364,86 @@ public class KinesisClientLibConfiguratorTest { /** * This credentials provider will always succeed */ - public static class AlwaysSucceedCredentialsProvider implements AwsCredentialsProvider { + public static class AlwaysSucceedCredentialsProvider implements AWSCredentialsProvider { @Override - public AwsCredentials resolveCredentials() { - return null; + public AWSCredentials getCredentials() { + return new BasicAWSCredentials("a", "b"); } + @Override + public void refresh() { + + } } /** * This credentials provider will always succeed */ - public static class AlwaysSucceedCredentialsProviderKinesis implements AwsCredentialsProvider { + public static class AlwaysSucceedCredentialsProviderKinesis implements AWSCredentialsProvider { @Override - public AwsCredentials resolveCredentials() { - return AwsBasicCredentials.create("", ""); + public AWSCredentials getCredentials() { + return new BasicAWSCredentials("", ""); } + @Override + public void refresh() { + + } } /** * This credentials provider will always succeed */ - public static class AlwaysSucceedCredentialsProviderDynamoDB implements AwsCredentialsProvider { + public static class AlwaysSucceedCredentialsProviderDynamoDB implements AWSCredentialsProvider { @Override - public AwsCredentials resolveCredentials() { - return AwsBasicCredentials.create("", ""); + public AWSCredentials getCredentials() { + return new BasicAWSCredentials("", ""); } + @Override + public void refresh() { + + } } /** * This credentials provider will always succeed */ - public static class AlwaysSucceedCredentialsProviderCloudWatch implements AwsCredentialsProvider { + public static class AlwaysSucceedCredentialsProviderCloudWatch implements AWSCredentialsProvider { @Override - public AwsCredentials resolveCredentials() { - return AwsBasicCredentials.create("", ""); + public AWSCredentials getCredentials() { + return new BasicAWSCredentials("", ""); } + @Override + public void refresh() { + + } } /** * This credentials provider will always fail */ - public static class AlwaysFailCredentialsProvider implements AwsCredentialsProvider { + public static class AlwaysFailCredentialsProvider implements AWSCredentialsProvider { @Override - public AwsCredentials resolveCredentials() { + public AWSCredentials getCredentials() { throw new IllegalArgumentException(); } + @Override + public void refresh() { + + } } - private KinesisClientLibConfiguration getConfiguration(String configString) { + private MultiLangDaemonConfiguration getConfiguration(String configString) { InputStream input = new ByteArrayInputStream(configString.getBytes()); - KinesisClientLibConfiguration config = configurator.getConfiguration(input); + MultiLangDaemonConfiguration config = configurator.getConfiguration(input); return config; } } diff --git a/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/MultiLangDaemonConfigurationTest.java b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/MultiLangDaemonConfigurationTest.java new file mode 100644 index 00000000..868d6d49 --- /dev/null +++ b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/MultiLangDaemonConfigurationTest.java @@ -0,0 +1,177 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.kinesis.multilang.config; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.junit.Assert.assertThat; + +import org.apache.commons.beanutils.BeanUtilsBean; +import org.apache.commons.beanutils.ConvertUtilsBean; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; + +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.kinesis.processor.ShardRecordProcessorFactory; +import software.amazon.kinesis.retrieval.fanout.FanOutConfig; +import software.amazon.kinesis.retrieval.polling.PollingConfig; + +@RunWith(MockitoJUnitRunner.class) +public class MultiLangDaemonConfigurationTest { + + private BeanUtilsBean utilsBean; + private ConvertUtilsBean convertUtilsBean; + + @Mock + private ShardRecordProcessorFactory shardRecordProcessorFactory; + + @Rule + public final ExpectedException thrown = ExpectedException.none(); + + @Before + public void setup() { + convertUtilsBean = new ConvertUtilsBean(); + utilsBean = new BeanUtilsBean(convertUtilsBean); + } + + public MultiLangDaemonConfiguration baseConfiguration() { + MultiLangDaemonConfiguration configuration = new MultiLangDaemonConfiguration(utilsBean, convertUtilsBean); + configuration.setApplicationName("Test"); + configuration.setStreamName("Test"); + configuration.getKinesisCredentialsProvider().set("class", DefaultCredentialsProvider.class.getName()); + + return configuration; + } + + @Test + public void testSetPrimitiveValue() { + MultiLangDaemonConfiguration configuration = baseConfiguration(); + configuration.setMaxLeasesForWorker(10); + + MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration + .resolvedConfiguration(shardRecordProcessorFactory); + + assertThat(resolvedConfiguration.leaseManagementConfig.maxLeasesForWorker(), equalTo(10)); + } + + @Test + public void testDefaultRetrievalConfig() { + MultiLangDaemonConfiguration configuration = baseConfiguration(); + + MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration + .resolvedConfiguration(shardRecordProcessorFactory); + + assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), + instanceOf(FanOutConfig.class)); + } + + @Test + public void testDefaultRetrievalConfigWithPollingConfigSet() { + MultiLangDaemonConfiguration configuration = baseConfiguration(); + configuration.setMaxRecords(10); + + MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration + .resolvedConfiguration(shardRecordProcessorFactory); + + assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), + instanceOf(PollingConfig.class)); + } + + @Test + public void testFanoutRetrievalMode() { + MultiLangDaemonConfiguration configuration = baseConfiguration(); + configuration.setRetrievalMode(RetrievalMode.FANOUT); + + MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration + .resolvedConfiguration(shardRecordProcessorFactory); + + assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), + instanceOf(FanOutConfig.class)); + } + + @Test + public void testPollingRetrievalMode() { + MultiLangDaemonConfiguration configuration = baseConfiguration(); + configuration.setRetrievalMode(RetrievalMode.POLLING); + + MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration + .resolvedConfiguration(shardRecordProcessorFactory); + + assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), + instanceOf(PollingConfig.class)); + } + + @Test + public void testRetrievalModeSetForPollingString() throws Exception { + MultiLangDaemonConfiguration configuration = baseConfiguration(); + + utilsBean.setProperty(configuration, "retrievalMode", RetrievalMode.POLLING.name().toLowerCase()); + + MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration + .resolvedConfiguration(shardRecordProcessorFactory); + + assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), + instanceOf(PollingConfig.class)); + } + + @Test + public void testRetrievalModeSetForFanoutString() throws Exception { + MultiLangDaemonConfiguration configuration = baseConfiguration(); + + utilsBean.setProperty(configuration, "retrievalMode", RetrievalMode.FANOUT.name().toLowerCase()); + + MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration + .resolvedConfiguration(shardRecordProcessorFactory); + + assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), + instanceOf(FanOutConfig.class)); + } + + @Test + public void testInvalidRetrievalMode() throws Exception { + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("Unknown retrieval type"); + + MultiLangDaemonConfiguration configuration = baseConfiguration(); + + utilsBean.setProperty(configuration, "retrievalMode", "invalid"); + } + + @Test + public void testFanoutConfigSetConsumerName() { + String consumerArn = "test-consumer"; + + MultiLangDaemonConfiguration configuration = baseConfiguration(); + + configuration.setRetrievalMode(RetrievalMode.FANOUT); + configuration.getFanoutConfig().setConsumerArn(consumerArn); + + MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration + .resolvedConfiguration(shardRecordProcessorFactory); + + assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), + instanceOf(FanOutConfig.class)); + FanOutConfig fanOutConfig = (FanOutConfig) resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(); + + assertThat(fanOutConfig.consumerArn(), equalTo(consumerArn)); + } + +} \ No newline at end of file diff --git a/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/PollingConfigBeanTest.java b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/PollingConfigBeanTest.java new file mode 100644 index 00000000..6ca0de63 --- /dev/null +++ b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/PollingConfigBeanTest.java @@ -0,0 +1,62 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.kinesis.multilang.config; + +import org.apache.commons.beanutils.BeanUtilsBean; +import org.apache.commons.beanutils.ConvertUtilsBean; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; +import software.amazon.awssdk.services.kinesis.KinesisAsyncClient; +import software.amazon.kinesis.retrieval.polling.PollingConfig; + +import java.util.Optional; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.junit.Assert.assertThat; + +@RunWith(MockitoJUnitRunner.class) +public class PollingConfigBeanTest { + + @Mock + private KinesisAsyncClient kinesisAsyncClient; + + @Test + public void testAllPropertiesTransit() { + PollingConfigBean pollingConfigBean = new PollingConfigBean(); + pollingConfigBean.setIdleTimeBetweenReadsInMillis(1000); + pollingConfigBean.setMaxGetRecordsThreadPool(20); + pollingConfigBean.setMaxRecords(5000); + pollingConfigBean.setRetryGetRecordsInSeconds(30); + + ConvertUtilsBean convertUtilsBean = new ConvertUtilsBean(); + BeanUtilsBean utilsBean = new BeanUtilsBean(convertUtilsBean); + + MultiLangDaemonConfiguration multiLangDaemonConfiguration = new MultiLangDaemonConfiguration(utilsBean, convertUtilsBean); + multiLangDaemonConfiguration.setStreamName("test-stream"); + + PollingConfig pollingConfig = pollingConfigBean.build(kinesisAsyncClient, multiLangDaemonConfiguration); + + assertThat(pollingConfig.kinesisClient(), equalTo(kinesisAsyncClient)); + assertThat(pollingConfig.streamName(), equalTo(multiLangDaemonConfiguration.getStreamName())); + assertThat(pollingConfig.idleTimeBetweenReadsInMillis(), equalTo(pollingConfigBean.getIdleTimeBetweenReadsInMillis())); + assertThat(pollingConfig.maxGetRecordsThreadPool(), equalTo(Optional.of(pollingConfigBean.getMaxGetRecordsThreadPool()))); + assertThat(pollingConfig.maxRecords(), equalTo(pollingConfigBean.getMaxRecords())); + assertThat(pollingConfig.retryGetRecordsInSeconds(), equalTo(Optional.of(pollingConfigBean.getRetryGetRecordsInSeconds()))); + } + +} \ No newline at end of file diff --git a/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/messages/JsonFriendlyRecordTest.java b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/messages/JsonFriendlyRecordTest.java new file mode 100644 index 00000000..9e19d7c7 --- /dev/null +++ b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/messages/JsonFriendlyRecordTest.java @@ -0,0 +1,167 @@ +/* + * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Amazon Software License (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/asl/ + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.kinesis.multilang.messages; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.CoreMatchers.sameInstance; +import static org.junit.Assert.assertThat; + +import java.nio.ByteBuffer; +import java.time.Instant; +import java.util.Arrays; +import java.util.List; +import java.util.function.Function; +import java.util.function.Supplier; + +import org.hamcrest.Description; +import org.hamcrest.Matcher; +import org.hamcrest.TypeSafeDiagnosingMatcher; +import org.junit.Test; + +import software.amazon.kinesis.retrieval.KinesisClientRecord; + +public class JsonFriendlyRecordTest { + + private KinesisClientRecord kinesisClientRecord; + + @Test + public void testRecordHandlesNullData() { + kinesisClientRecord = defaultRecord().data(null).build(); + JsonFriendlyRecord jsonFriendlyRecord = JsonFriendlyRecord.fromKinesisClientRecord(kinesisClientRecord); + + assertThat(jsonFriendlyRecord, equivalentTo(kinesisClientRecord)); + } + + @Test + public void testRecordHandlesNoByteArrayBuffer() { + byte[] expected = new byte[] { 1, 2, 3, 4 }; + + ByteBuffer expectedBuffer = ByteBuffer.allocateDirect(expected.length); + + expectedBuffer.put(expected); + expectedBuffer.rewind(); + + kinesisClientRecord = defaultRecord().data(expectedBuffer).build(); + JsonFriendlyRecord jsonFriendlyRecord = JsonFriendlyRecord.fromKinesisClientRecord(kinesisClientRecord); + + expectedBuffer.rewind(); + assertThat(jsonFriendlyRecord, equivalentTo(kinesisClientRecord)); + } + + @Test + public void testRecordHandlesArrayByteBuffer() { + ByteBuffer expected = ByteBuffer.wrap(new byte[] { 1, 2, 3, 4 }); + kinesisClientRecord = defaultRecord().data(expected).build(); + JsonFriendlyRecord jsonFriendlyRecord = JsonFriendlyRecord.fromKinesisClientRecord(kinesisClientRecord); + + assertThat(jsonFriendlyRecord, equivalentTo(kinesisClientRecord)); + } + + private static RecordMatcher equivalentTo(KinesisClientRecord expected) { + return new RecordMatcher(expected); + } + + private static class RecordMatcher extends TypeSafeDiagnosingMatcher { + + private final KinesisClientRecord expected; + private final List> matchers; + + private RecordMatcher(KinesisClientRecord expected) { + this.matchers = Arrays.asList( + new FieldMatcher<>("approximateArrivalTimestamp", + equalTo(expected.approximateArrivalTimestamp().toEpochMilli()), + JsonFriendlyRecord::getApproximateArrivalTimestamp), + new FieldMatcher<>("partitionKey", expected::partitionKey, JsonFriendlyRecord::getPartitionKey), + new FieldMatcher<>("sequenceNumber", expected::sequenceNumber, + JsonFriendlyRecord::getSequenceNumber), + new FieldMatcher<>("subSequenceNumber", expected::subSequenceNumber, + JsonFriendlyRecord::getSubSequenceNumber), + new FieldMatcher<>("data", dataEquivalentTo(expected.data()), JsonFriendlyRecord::getData)); + + this.expected = expected; + } + + @Override + protected boolean matchesSafely(JsonFriendlyRecord item, Description mismatchDescription) { + return matchers.stream().map(m -> { + if (!m.matches(item)) { + m.describeMismatch(item, mismatchDescription); + return false; + } + return true; + }).reduce((l, r) -> l && r).orElse(true); + } + + @Override + public void describeTo(Description description) { + description.appendText("A JsonFriendlyRecord matching ").appendList("(", ", ", ")", matchers); + } + } + + private static Matcher dataEquivalentTo(ByteBuffer expected) { + if (expected == null) { + return nullValue(); + } else { + if (expected.hasArray()) { + return sameInstance(expected.array()); + } else { + byte[] contents = new byte[expected.limit()]; + expected.get(contents); + return equalTo(contents); + } + } + } + + private static class FieldMatcher extends TypeSafeDiagnosingMatcher { + + final String fieldName; + final Matcher matcher; + final Function extractor; + + private FieldMatcher(String fieldName, Supplier expected, Function extractor) { + this(fieldName, equalTo(expected.get()), extractor); + } + + private FieldMatcher(String fieldName, Matcher matcher, Function extractor) { + this.fieldName = fieldName; + this.matcher = matcher; + this.extractor = extractor; + } + + @Override + protected boolean matchesSafely(ClassT item, Description mismatchDescription) { + ItemT actual = extractor.apply(item); + if (!matcher.matches(actual)) { + mismatchDescription.appendText(fieldName).appendText(": "); + matcher.describeMismatch(actual, mismatchDescription); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText(fieldName).appendText(": ").appendDescriptionOf(matcher); + } + } + + private KinesisClientRecord.KinesisClientRecordBuilder defaultRecord() { + return KinesisClientRecord.builder().partitionKey("test-partition").sequenceNumber("123") + .approximateArrivalTimestamp(Instant.now()); + } + +} \ No newline at end of file diff --git a/amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/messages/MessageTest.java b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/messages/MessageTest.java similarity index 83% rename from amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/messages/MessageTest.java rename to amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/messages/MessageTest.java index 179c4ad8..217304b9 100644 --- a/amazon-kinesis-client-multilang/src/test/java/com/amazonaws/services/kinesis/multilang/messages/MessageTest.java +++ b/amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/messages/MessageTest.java @@ -12,7 +12,7 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ -package com.amazonaws.services.kinesis.multilang.messages; +package software.amazon.kinesis.multilang.messages; import java.nio.ByteBuffer; import java.util.Collections; @@ -26,6 +26,13 @@ import com.fasterxml.jackson.databind.ObjectMapper; import software.amazon.kinesis.lifecycle.events.InitializationInput; import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput; import software.amazon.kinesis.lifecycle.ShutdownReason; +import software.amazon.kinesis.multilang.messages.CheckpointMessage; +import software.amazon.kinesis.multilang.messages.InitializeMessage; +import software.amazon.kinesis.multilang.messages.Message; +import software.amazon.kinesis.multilang.messages.ProcessRecordsMessage; +import software.amazon.kinesis.multilang.messages.ShutdownMessage; +import software.amazon.kinesis.multilang.messages.ShutdownRequestedMessage; +import software.amazon.kinesis.multilang.messages.StatusMessage; import software.amazon.kinesis.retrieval.KinesisClientRecord; public class MessageTest { @@ -47,7 +54,9 @@ public class MessageTest { new StatusMessage("processRecords"), new InitializeMessage(), new ProcessRecordsMessage(), - new ShutdownRequestedMessage() + new ShutdownRequestedMessage(), + new LeaseLostMessage(), + new ShardEndedMessage() }; // TODO: fix this diff --git a/amazon-kinesis-client-multilang/src/test/resources/logback.xml b/amazon-kinesis-client-multilang/src/test/resources/logback-test.xml similarity index 100% rename from amazon-kinesis-client-multilang/src/test/resources/logback.xml rename to amazon-kinesis-client-multilang/src/test/resources/logback-test.xml diff --git a/amazon-kinesis-client/pom.xml b/amazon-kinesis-client/pom.xml index 1b791739..e97e9b4c 100644 --- a/amazon-kinesis-client/pom.xml +++ b/amazon-kinesis-client/pom.xml @@ -20,7 +20,7 @@ software.amazon.kinesis amazon-kinesis-client-pom - 2.0.5 + 2.1.0 amazon-kinesis-client @@ -45,8 +45,6 @@ - 1.11.272 - 2.0.6 1.0.392 libsqlite4java ${project.build.directory}/test-lib @@ -87,7 +85,7 @@ org.apache.commons commons-lang3 - 3.7 + 3.8.1 org.slf4j diff --git a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/KinesisClientUtil.java b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/KinesisClientUtil.java index 634c9b01..02ac9554 100644 --- a/amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/KinesisClientUtil.java +++ b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/KinesisClientUtil.java @@ -31,7 +31,10 @@ public class KinesisClientUtil { * @return */ public static KinesisAsyncClient createKinesisAsyncClient(KinesisAsyncClientBuilder clientBuilder) { - return clientBuilder.httpClientBuilder(NettyNioAsyncHttpClient.builder().maxConcurrency(Integer.MAX_VALUE)) - .build(); + return adjustKinesisClientBuilder(clientBuilder).build(); + } + + public static KinesisAsyncClientBuilder adjustKinesisClientBuilder(KinesisAsyncClientBuilder builder) { + return builder.httpClientBuilder(NettyNioAsyncHttpClient.builder().maxConcurrency(Integer.MAX_VALUE)); } } diff --git a/pom.xml b/pom.xml index 7c5cdc7b..5e725ae1 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ amazon-kinesis-client-pom pom Amazon Kinesis Client Library - 2.0.5 + 2.1.0 The Amazon Kinesis Client Library for Java enables Java developers to easily consume and process data from Amazon Kinesis. @@ -31,7 +31,7 @@ - 1.11.272 + 2.2.0